{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost\n",
    "import lightgbm\n",
    "import catboost\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report, f1_score,precision_score,recall_score,accuracy_score\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import pandas as pd\n",
    "import gc\n",
    "from sklearn import feature_extraction\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier, \n",
    "                              GradientBoostingClassifier, ExtraTreesClassifier)\n",
    "from category_encoders.woe import WOEEncoder\n",
    "from category_encoders.target_encoder import TargetEncoder\n",
    "from category_encoders.cat_boost import CatBoostEncoder\n",
    "from category_encoders.ordinal import OrdinalEncoder\n",
    "from category_encoders.leave_one_out import LeaveOneOutEncoder\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "plt.rcParams['font.sans-serif']=['Simhei']\n",
    "plt.rcParams['axes.unicode_minus']=False\n",
    "import json\n",
    "import jieba\n",
    "from scipy.stats import entropy\n",
    "import fasttext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## WOE编码\n",
    "要对一个变量进行WOE编码，需要首先把这个变量进行分组处理（也叫离散化、分箱等等）\n",
    "\n",
    "WOE_i=ln(py_i)/ln(pn_i)=ln(sum(y_pos)/y_T)/ln(sum(y_neg)/y_T)\n",
    "\n",
    "WOE表示的实际上是“当前分组中响应客户占所有响应客户的比例”和“当前分组中没有响应的客户占所有没有响应的客户的比例”的差异。\n",
    "\n",
    "WOE也可以这么理解，他表示的是当前这个组中响应的客户和未响应客户的比值，和所有样本中这个比值的差异。\n",
    "\n",
    "这个差异是用这两个比值的比值，再取对数来表示的。\n",
    "\n",
    "WOE越大，这种差异越大，这个分组里的样本响应的可能性就越大，WOE越小，差异越小，这个分组里的样本响应的可能性就越小\n",
    "\n",
    "目标编码\n",
    "\n",
    "## Target encoding\n",
    "是表示分类列的一种非常有效的方法，并且仅占用一个特征空间，也称为均值编码。\n",
    "\n",
    "该列中的每个值都被该类别的平均目标值替代。这可以更直接地表示分类变量和目标变量之间的关系，\n",
    "\n",
    "并且也是一种很受欢迎的技术方法（尤其是在 Kaggle 比赛中）。\n",
    "\n",
    "## 留一法（Leave-one-out）编码\n",
    "试图通过计算平均值（不包括当前行值）来弥补对 y 变量的依赖以及值的多样性。这使异常值的影响趋于平稳，并创建更多样化的编码值。\n",
    "\n",
    "\n",
    "由于模型不仅要面对每个编码类的相同值，还要面对一个范围值，因此它可以更好地泛化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "#该函数用于返回一个dataframe中哪些是object类型\n",
    "def get_object_columns(df):\n",
    "    return df.select_dtypes(include=[\"object\"]).columns.tolist()\n",
    "#该函数用于计算一个dataframe中每一列的空值比例,thresh为阈值，返回一个列名的列表\n",
    "def get_nan_ratio(df,thresh):\n",
    "    nan_dic=(df.isnull().sum()/len(base_info)).to_dict()\n",
    "    nan_list=[]\n",
    "    for key in nan_dic.keys():\n",
    "        if nan_dic[key]>thresh:#缺失值高于thresh的列被返回\n",
    "            nan_list.append(key)\n",
    "    return nan_list,sorted( nan_dic.items(),key = lambda x:x[1],reverse = True)\n",
    "#........................分箱.................................\n",
    "def bucket(df,name,bucket_len):\n",
    "    gap_list=[df[name].quantile(i/bucket_len) for i in range(bucket_len+1)]#以分位数作为分箱标志\n",
    "    len_data=len(df[name])\n",
    "    new_col=[]\n",
    "    for i in df[name].values:\n",
    "        for j in range(len(gap_list)):\n",
    "            if gap_list[j]>=i:\n",
    "                encode=j\n",
    "                break\n",
    "        new_col.append(encode)\n",
    "    return new_col\n",
    "#作两个特征的交叉\n",
    "def cross_two(df,name_1,name_2):\n",
    "    new_col=[]\n",
    "    encode=0\n",
    "    dic={}\n",
    "    val_1=df[name_1]\n",
    "    val_2=df[name_2]\n",
    "    for i in tqdm(range(len(val_1))):\n",
    "        tmp=str(val_1[i])+'_'+str(val_2[i])\n",
    "        if tmp in dic:\n",
    "            new_col.append(dic[tmp])\n",
    "        else:\n",
    "            dic[tmp]=encode\n",
    "            new_col.append(encode)\n",
    "            encode+=1\n",
    "    return new_col\n",
    "# 特征nunique分布\n",
    "def nunique_info(df,cat_features):\n",
    "    for cat_fea in cat_features:\n",
    "        print(cat_fea + \"的特征分布如下：\")\n",
    "        print(\"{}特征有个{}不同的值\".format(cat_fea, df[cat_fea].nunique()))\n",
    "        print(df[cat_fea].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "class Category_encoders_hepler(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    #\n",
    "    def encode(self,data_train,data_test,cols,mode):\n",
    "        if mode==\"catboost\":\n",
    "            encoder = CatBoostEncoder(cols=cols, \n",
    "                            handle_unknown='value',  \n",
    "                            handle_missing='value',\n",
    "                            ).fit(data_train[cols],data_train['label'])\n",
    "        elif mode==\"woe\":\n",
    "            encoder = WOEEncoder(cols=cols, \n",
    "                                handle_unknown='value',  \n",
    "                                handle_missing='value',\n",
    "                                ).fit(data_train[cols],data_train['label'])\n",
    "        elif mode==\"target\":\n",
    "            encoder = TargetEncoder(cols=cols, \n",
    "                            handle_unknown='value',  \n",
    "                            handle_missing='value',\n",
    "                            min_samples_leaf=10).fit(data_train[cols],data_train['label'])\n",
    "            \n",
    "        elif mode==\"leave_one_out\":\n",
    "            encoder = LeaveOneOutEncoder(cols=cols, \n",
    "                            handle_unknown='value',  \n",
    "                            handle_missing='value',\n",
    "                           ).fit(data_train[cols],data_train['label'])\n",
    "        elif mode==\"ordinal\":\n",
    "            encoder = OrdinalEncoder(cols = cols, \n",
    "                         handle_unknown = 'value', \n",
    "                         handle_missing = 'value').fit(data_train[cols],None)\n",
    "        \n",
    "        suffix='_'+mode.split('_')[0]\n",
    "        encoded_train = encoder.transform(data_train[cols]) # 转换训练集\n",
    "        encoded_train.columns=[i+suffix for i in encoded_train.columns]\n",
    "        data_train=pd.concat([data_train,encoded_train],axis=1)\n",
    "\n",
    "        encoded_test = encoder.transform(data_test[cols]) # 转换测试集\n",
    "        encoded_test.columns=[i+suffix for i in encoded_test.columns]\n",
    "        data_test=pd.concat([data_test,encoded_test],axis=1)\n",
    "        \n",
    "        data_train=data_train.drop(cols,axis=1)\n",
    "        data_test=data_test.drop(cols,axis=1)\n",
    "        return data_train,data_test\n",
    "#------一个验证编码正确性的小例子---------------\n",
    "# encoder_tools=Category_encoders_hepler()\n",
    "# example_train_df = pd.DataFrame(np.array([['male',10],['female', 20], ['male',10], \n",
    "#                        ['female',20],['female',15]]),\n",
    "#              columns = ['Sex','Type'])\n",
    "# example_train_df['label'] = np.array([False, True, True, False, False])\n",
    "\n",
    "# # 随机生成一些测试集, 并有意让其包含未在训练集出现过的类别与缺失值\n",
    "# example_test_df = pd.DataFrame(np.array([['female',20],['male', 20], ['others',15], \n",
    "#                        ['male',20],['female',40], ['male', 25]]),\n",
    "#              columns = ['Sex','Type'])\n",
    "# example_test_df.loc[4,'Type'] = np.nan\n",
    "# train_new,test_new=encode_tool.encode(example_train_df,example_test_df,['Sex','Type'],mode=\"ordinal\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0 数据的简单分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "base_info shape: (24865, 33) id unique: 24865\n",
      "annual_report_info shape: (22550, 23) id unique: 8937\n",
      "tax_info shape: (29195, 9) id unique: 808\n",
      "change_info shape: (45940, 5) id unique: 8726\n",
      "news_info shape: (10518, 3) id unique: 927\n",
      "other_info shape: (1890, 4) id unique: 1888\n",
      "entprise_info shape: (14865, 2) id unique: 14865\n",
      "entprise_evaluate shape: (10000, 2) id unique: 10000\n"
     ]
    }
   ],
   "source": [
    "base_info=pd.read_csv('../train/base_info.csv')#企业的基本信息\n",
    "annual_report_info=pd.read_csv('../train/annual_report_info.csv')#企业的年报基本信息\n",
    "tax_info=pd.read_csv('../train/tax_info.csv')#企业的纳税信息\n",
    "change_info=pd.read_csv('../train/change_info.csv')#变更信息\n",
    "news_info=pd.read_csv('../train/news_info.csv')#舆情信息\n",
    "other_info=pd.read_csv('../train/other_info.csv')#其它信息\n",
    "entprise_info=pd.read_csv('../train/entprise_info.csv')#企业标注信息{0: 13884, 1: 981}\n",
    "entprise_evaluate=pd.read_csv('../entprise_evaluate.csv')#未标注信息\n",
    "\n",
    "print('base_info shape:',base_info.shape,'id unique:',len(base_info['id'].unique()))\n",
    "print('annual_report_info shape:',annual_report_info.shape,'id unique:',len(annual_report_info['id'].unique()))\n",
    "print('tax_info shape:',tax_info.shape,'id unique:',len(tax_info['id'].unique()))\n",
    "print('change_info shape:',change_info.shape,'id unique:',len(change_info['id'].unique()))\n",
    "print('news_info shape:',news_info.shape,'id unique:',len(news_info['id'].unique()))\n",
    "print('other_info shape:',other_info.shape,'id unique:',len(other_info['id'].unique()))\n",
    "print('entprise_info shape:',entprise_info.shape,'id unique:',len(entprise_info['id'].unique()))\n",
    "print('entprise_evaluate shape:',entprise_evaluate.shape,'id unique:',len(entprise_evaluate['id'].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 特征构建 \n",
    "###  tfidf处理经营范围(opscope)特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "对opscope提取tfidif特征完毕..........\n"
     ]
    }
   ],
   "source": [
    "# tfidif 处理经营范围的特征\n",
    "#cn_stopwords.txt来源于 https://github.com/goto456/stopwords\n",
    "# 创建一个停用词列表\n",
    "def tfidif_frt():\n",
    "    stopwords = stopwords = [line.strip() for line in open('D:/tianma/stopwords-master/cn_stopwords.txt',encoding='UTF-8').readlines()]\n",
    "    stopwords+=['、', '；', '，', '）','（']\n",
    "    #\n",
    "    train_df_scope=base_info.merge(entprise_info)[['id','opscope','label']]\n",
    "    test_df_scope=base_info[base_info['id'].isin(entprise_evaluate['id'].unique().tolist())]\n",
    "    test_df_scope=test_df_scope.reset_index(drop=True)[['id','opscope']]\n",
    "    str_label_0=''\n",
    "    str_label_1=''\n",
    "    for index,name,opscope,label in train_df_scope.itertuples():\n",
    "        # 结巴分词\n",
    "        seg_text = jieba.cut(opscope.replace(\"\\t\", \" \").replace(\"\\n\", \" \"))\n",
    "        outline = \" \".join(seg_text)\n",
    "        out_str=\"\"\n",
    "        for per in outline.split():\n",
    "            if per not in stopwords: \n",
    "                out_str += per\n",
    "                out_str+=\" \"\n",
    "        if label==0:\n",
    "            str_label_0+=out_str\n",
    "        else:\n",
    "            str_label_1+=out_str\n",
    "    corpus=[str_label_0,str_label_1]\n",
    "    vectorizer=CountVectorizer()#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频\n",
    "    transformer=TfidfTransformer()#该类会统计每个词语的tf-idf权值\n",
    "    tfidf=transformer.fit_transform(vectorizer.fit_transform(corpus))#第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵\n",
    "    word=vectorizer.get_feature_names()#获取词袋模型中的所有词语总共7175个词语\n",
    "    weight=tfidf.toarray()#将(2, 7175)tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重\n",
    "    # for i in range(len(weight)):#打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重\n",
    "    #     #\n",
    "    #     for j in range(len(word)):\n",
    "    #         print(word[j],weight[i][j])\n",
    "    #下面将会根据tfidi算出来的权重将经营范围的文本特征转换为数值(利用weight[1,:]也即各个词语在第二类(违法类中所占据的权重之和))\n",
    "    illegal_word_weights={}\n",
    "    for i in range(len(word)):\n",
    "        illegal_word_weights[word[i]]=weight[1][i]\n",
    "    tfidi_opscope=[]\n",
    "    for index,name,opscope in base_info[['id','opscope']].itertuples():\n",
    "        # \n",
    "        seg_text = jieba.cut(opscope.replace(\"\\t\", \" \").replace(\"\\n\", \" \"))\n",
    "        outline = \" \".join(seg_text)\n",
    "        tfidi_frt=0\n",
    "        for per in outline.split():\n",
    "            if per in illegal_word_weights: \n",
    "                tfidi_frt+=illegal_word_weights[per]\n",
    "        tfidi_opscope.append(tfidi_frt)\n",
    "    return tfidi_opscope\n",
    "base_info['tfidif_opscope']=tfidif_frt()\n",
    "base_info.drop(['opscope'],axis=1,inplace=True)\n",
    "print('对opscope提取tfidif特征完毕..........')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  change_info、other_info，news_info，annual_report_info,tax表格的简单特征构建"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- id:企业唯一标识(object)\n",
    "- oplocdistrict:行政区划代码\n",
    "- industryphy:行业类别代码(object)\n",
    "- industryco:行业细类代码\n",
    "- dom:经营地址(object)\n",
    "- opscope:经营范围(object)\n",
    "- enttype:企业类型\n",
    "- enttypeitem:企业类型小类\n",
    "- opfrom:经营期限起(object)\n",
    "- opto:经营期限止(object)\n",
    "- state:状态\n",
    "- orgid:机构标识\n",
    "- jobid:职位标识\n",
    "- adbusign:是否广告经营\n",
    "- townsign:是否城镇\n",
    "- regtype:主题登记类型\n",
    "- empnum:从业人数\n",
    "- compform:组织形式\n",
    "- parnum:合伙人数\n",
    "- exenum:执行人数\n",
    "- opform:经营方式(object)\n",
    "- ptbusscope:兼营范围\n",
    "- venind:风险行业\n",
    "- enttypeminu:企业类型细类\n",
    "- midpreindcode:中西部优势产业代码\n",
    "- protype:项目类型\n",
    "- oploc:经营场所(object)\n",
    "- regcap:注册资本（金）\n",
    "- reccap:实缴资本\n",
    "- forreccap:实缴资本（外方）\n",
    "- forregcap:注册资本（外方）\n",
    "- congro:投资总额\n",
    "- enttypegb:企业（机构）类型]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id的unique长度:24865\n",
      "industryphy的unique长度:20\n",
      "industryco的unique长度:346\n",
      "enttype的unique长度:17\n",
      "enttypeitem的unique长度:32\n",
      "opform的unique长度:34\n",
      "oploc的unique长度:5351\n",
      "id_prefix的unique长度:65\n",
      "dom_prefix的unique长度:412\n",
      "id_dom的unique长度:24865\n",
      "industryphy_industryco的unique长度:348\n",
      "enttype_enttypeitem的unique长度:37\n",
      "industryphy_enttype的unique长度:99\n",
      "enttype_enttypeitem_industryphy_industryco的unique长度:1072\n",
      "industryphy_id_prefix的unique长度:245\n",
      "enttype_id_prefix的unique长度:116\n",
      "industryphy_industryco_id_prefix的unique长度:1680\n",
      "enttype_enttypeitem_id_prefix的unique长度:194\n",
      "industryphy_dom_prefix的unique长度:773\n",
      "enttype_dom_prefix的unique长度:597\n",
      "enttypegb_industryphy的unique长度:250\n",
      "enttypegb_enttype的unique长度:53\n",
      "enttypegb_industryphy_enttype的unique长度:250\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(24865, 45)"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#可以看到，经营地址编码虽然不同，但是长度变化却很小.总共长度只有21种，并且都可以被16整除，前32位不同的只有412种\n",
    "#对于id特征亦如此，长度都为48，所以可以先对id按16位进行切分\n",
    "#pd.Series([len(i) for i in base_info['dom'].values]).unique()/16\n",
    "base_info['id_prefix']=[per[:16] for per in base_info['id'].values]\n",
    "base_info['dom_prefix']=[per[:32] for per in base_info['dom'].values]\n",
    "base_info['id_dom']=base_info['id'].astype(str) + '_' + base_info['dom'].astype(str)\n",
    "#-------------------------------------\n",
    "#构建交叉特征\n",
    "base_info['enttypegb']=base_info['enttypegb'].fillna(\"无\")\n",
    "base_info['enttype']=base_info['enttype'].astype(str)\n",
    "base_info['enttype']=base_info['enttype'].fillna(\"无\")\n",
    "base_info['enttypeitem']=base_info['enttypeitem'].fillna(\"无\")\n",
    "base_info['industryphy']=base_info['industryphy'].fillna(\"无\")\n",
    "base_info['industryco']=base_info['industryco'].fillna(\"无\")\n",
    "#行业类别_行业细类\n",
    "base_info['industryphy_industryco']=base_info['industryphy'].astype(str) + '_' + base_info['industryco'].astype(str)\n",
    "#企业类型_企业类型小类\n",
    "base_info['enttype_enttypeitem']=base_info['enttype'].astype(str) + '_' + base_info['enttypeitem'].astype(str)\n",
    "#行业类别_企业类型\n",
    "base_info['industryphy_enttype']=base_info['industryphy'].astype(str) + '_' + base_info['enttype'].astype(str)\n",
    "#\n",
    "base_info['enttype_enttypeitem_industryphy_industryco']=base_info['enttype_enttypeitem'].astype(str) + '_' + base_info['industryphy_industryco'].astype(str)\n",
    "#\n",
    "base_info['industryphy_id_prefix']=base_info['industryphy'].astype(str) + '_' + base_info['id_prefix'].astype(str)\n",
    "base_info['enttype_id_prefix']=base_info['enttype'].astype(str) + '_' + base_info['id_prefix'].astype(str)\n",
    "base_info['industryphy_industryco_id_prefix']=base_info['industryphy_industryco'].astype(str) + '_' + base_info['id_prefix'].astype(str)\n",
    "base_info['enttype_enttypeitem_id_prefix']=base_info['enttype_enttypeitem'].astype(str) + '_' + base_info['id_prefix'].astype(str)\n",
    "#\n",
    "base_info['industryphy_dom_prefix']=base_info['industryphy'].astype(str) + '_' + base_info['dom_prefix'].astype(str)\n",
    "base_info['enttype_dom_prefix']=base_info['enttype'].astype(str) + '_' + base_info['dom_prefix'].astype(str)\n",
    "#\n",
    "base_info['enttypegb_industryphy']=base_info['enttypegb'].astype(str) + '_' + base_info['industryphy'].astype(str)\n",
    "base_info['enttypegb_enttype']=base_info['enttypegb'].astype(str) + '_' + base_info['enttype'].astype(str)\n",
    "base_info['enttypegb_industryphy_enttype']=base_info['enttypegb'].astype(str) + '_' + base_info['industryphy_enttype'].astype(str)\n",
    "#---------------------------\n",
    "# #处理base_info数据\n",
    "base_info['reccap']=base_info['reccap'].fillna(-2)\n",
    "drop_nan_columns=get_nan_ratio(base_info,thresh=0.8)[0]#缺失值高于thresh=0.8直接删掉\n",
    "base_info_clean=base_info.drop(drop_nan_columns,axis=1)\n",
    "#'enttypeminu', 0.7076211542328574),企业类型细类\n",
    "#  ('venind', 0.6606877136537301),风险行业\n",
    "#  ('opto', 0.6450834506334204),\n",
    "#  ('opform', 0.638045445405188),\n",
    "#  ('compform', 0.5724512366780615),\n",
    "base_info_clean['null_enttypeminu']=(base_info_clean['enttypeminu'].isnull()).astype(int)\n",
    "base_info_clean['null_enttypeminu']=(base_info_clean['enttypeminu'].isnull()).astype(int)\n",
    "base_info_clean['venind']=base_info_clean['venind'].fillna(-2)\n",
    "base_info_clean['compform']=base_info_clean['compform'].fillna(-2)\n",
    "#------------------\n",
    "#\n",
    "base_info_clean['opfrom']=pd.to_datetime(base_info_clean['opfrom'])\n",
    "current_time=pd.to_datetime('2020-11-17 00:00:00')\n",
    "base_info_clean['alive_year']=(current_time-pd.to_datetime(base_info_clean['opfrom'])).dt.total_seconds()//3600//24//30\n",
    "base_info_clean.drop(['opfrom','opto','dom'],axis=1,inplace=True)\n",
    "#\n",
    "for i in get_object_columns(base_info_clean):\n",
    "    print(\"{}的unique长度:{}\".format(i,len(base_info_clean[i].unique())))\n",
    "\n",
    "#--------------------------分箱---------------------\n",
    "#注册资本减去实缴资本\n",
    "base_info_clean['regcap_diff_reccap']=base_info_clean['regcap']-base_info_clean['reccap']\n",
    "base_info_clean['regcap_div_reccap']=base_info_clean['reccap']/base_info_clean['regcap']\n",
    "#注册资本分箱\n",
    "base_info_clean['regcap']=base_info_clean['regcap'].fillna(base_info_clean['regcap'].median())\n",
    "base_info_clean['bucket_regcap']=pd.qcut(base_info_clean['regcap'], 10, labels=False,duplicates='drop')\n",
    "#实缴资本分箱\n",
    "base_info_clean['reccap']=base_info_clean['reccap'].fillna(base_info_clean['reccap'].median())\n",
    "base_info_clean['bucket_reccap']=pd.qcut(base_info_clean['reccap'], 10, labels=False,duplicates='drop')\n",
    "#注册资本_实缴资本分箱\n",
    "base_info_clean['regcap_diff_reccap']=base_info_clean['regcap_diff_reccap'].fillna(base_info_clean['regcap_diff_reccap'].median())\n",
    "base_info_clean['bucket_regcap_diff_reccap']=pd.qcut(base_info_clean['regcap_diff_reccap'], 10, labels=False,duplicates='drop')\n",
    "#\n",
    "base_info_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#以下各个类别对企业id做计数特征\n",
    "#industryphy:行业类别代码\n",
    "#industryco:行业细类代码\n",
    "#dom:经营地址(object)\n",
    "#opscope:经营范围(object)\n",
    "#enttype:企业类型\n",
    "#enttypeitem:企业类型小类\n",
    "'''\n",
    "count特征，选一个主键，如果选industryphy:行业类别代码做主键，与id做unique特征，则表示\n",
    "某一个具体的行业类别对应了多少个企业。\n",
    "'''\n",
    "def gen_cnt_feature(df, feature):\n",
    "    cnt_features = []\n",
    "    for fea in feature:\n",
    "        df[fea + '_count'] = df.groupby([fea])['id'].transform('count')\n",
    "        cnt_features.append(fea + '_count')\n",
    "    return df\n",
    "'''\n",
    "unique特征，选一个主键，如果选industryphy:行业类别代码做主键，与id做unique特征，则表示\n",
    "某一个具体的行业类别对应了多少独有的id,如果与industryco:行业细类代码做unique特征，则表示某一个具体的行业类别对应\n",
    "了多少个独有的行业细类别.\n",
    "'''\n",
    "def get_nunique(df,group_col,sta_col):\n",
    "    for i in group_col:\n",
    "        for j in sta_col:\n",
    "            if i!=j:\n",
    "                df[i+'_nunique_'+j] = df.groupby([i])[j].transform('nunique')\n",
    "                #df[i+'_mode_'+j] = df.groupby([i])[j].transform(lambda x :x.mode()[0])\n",
    "                #df[i+'_entropy_'+j] = df.groupby([i])[j].transform(lambda x: entropy(x.value_counts() / x.shape[0]))\n",
    "    return df\n",
    "#选取以下列作count特征\n",
    "cnt_cols=['enttypegb','enttype','enttypeitem','industryphy','industryco','industryphy_industryco',\n",
    "        'enttype_enttypeitem','industryphy_enttype','enttype_enttypeitem_industryphy_industryco',\n",
    "         'enttypegb_enttype','enttypegb_industryphy']\n",
    "base_info_clean=gen_cnt_feature(base_info_clean.copy(),cnt_cols)\n",
    "group_col=['enttypegb','industryphy']\n",
    "sta_col=['id','enttype','industryco','enttypeitem']\n",
    "base_info_clean=get_nunique(base_info_clean.copy(),group_col,sta_col)\n",
    "base_info_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>oplocdistrict</th>\n",
       "      <th>industryphy</th>\n",
       "      <th>industryco</th>\n",
       "      <th>enttype</th>\n",
       "      <th>enttypeitem</th>\n",
       "      <th>state</th>\n",
       "      <th>orgid</th>\n",
       "      <th>jobid</th>\n",
       "      <th>adbusign</th>\n",
       "      <th>...</th>\n",
       "      <th>enttypegb_industryphy</th>\n",
       "      <th>enttypegb_enttype</th>\n",
       "      <th>enttypegb_industryphy_enttype</th>\n",
       "      <th>null_enttypeminu</th>\n",
       "      <th>alive_year</th>\n",
       "      <th>regcap_diff_reccap</th>\n",
       "      <th>regcap_div_reccap</th>\n",
       "      <th>bucket_regcap</th>\n",
       "      <th>bucket_reccap</th>\n",
       "      <th>bucket_regcap_diff_reccap</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47645761dc56bb8c5fae00114b768b5d9b6e917c3aec07c4</td>\n",
       "      <td>340223</td>\n",
       "      <td>M</td>\n",
       "      <td>7513</td>\n",
       "      <td>1100</td>\n",
       "      <td>1150</td>\n",
       "      <td>6</td>\n",
       "      <td>340223010010000000</td>\n",
       "      <td>340200000000115392</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1151_M</td>\n",
       "      <td>1151_1100</td>\n",
       "      <td>1151_M_1100</td>\n",
       "      <td>0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3</td>\n",
       "      <td>340222</td>\n",
       "      <td>O</td>\n",
       "      <td>8090</td>\n",
       "      <td>9600</td>\n",
       "      <td>无</td>\n",
       "      <td>6</td>\n",
       "      <td>340222060010000000</td>\n",
       "      <td>340200000000112114</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9600_O</td>\n",
       "      <td>9600_9600</td>\n",
       "      <td>9600_O_9600</td>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>59b38c56de3836838082cfcb1a298951abfe15e6940c49ba</td>\n",
       "      <td>340202</td>\n",
       "      <td>R</td>\n",
       "      <td>9053</td>\n",
       "      <td>1100</td>\n",
       "      <td>1150</td>\n",
       "      <td>6</td>\n",
       "      <td>340202010010000000</td>\n",
       "      <td>400000000000753910</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1151_R</td>\n",
       "      <td>1151_1100</td>\n",
       "      <td>1151_R_1100</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e9f7b28ec10e047000d16ab79e1b5e6da434a1697cce7818</td>\n",
       "      <td>340221</td>\n",
       "      <td>L</td>\n",
       "      <td>7212</td>\n",
       "      <td>4500</td>\n",
       "      <td>4540</td>\n",
       "      <td>6</td>\n",
       "      <td>340221010010000000</td>\n",
       "      <td>400000000000013538</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4540_L</td>\n",
       "      <td>4540_4500</td>\n",
       "      <td>4540_L_4500</td>\n",
       "      <td>1</td>\n",
       "      <td>62.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f000950527a6feb63ee1ce82bb22ddd1ab8b8fdffa3b91fb</td>\n",
       "      <td>340202</td>\n",
       "      <td>R</td>\n",
       "      <td>8810</td>\n",
       "      <td>1100</td>\n",
       "      <td>1130</td>\n",
       "      <td>7</td>\n",
       "      <td>340200000000000000</td>\n",
       "      <td>400000000000283237</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1130_R</td>\n",
       "      <td>1130_1100</td>\n",
       "      <td>1130_R_1100</td>\n",
       "      <td>1</td>\n",
       "      <td>36.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24860</th>\n",
       "      <td>f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325</td>\n",
       "      <td>340225</td>\n",
       "      <td>O</td>\n",
       "      <td>8131</td>\n",
       "      <td>9600</td>\n",
       "      <td>无</td>\n",
       "      <td>6</td>\n",
       "      <td>340200000000116780</td>\n",
       "      <td>341400000000011622</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9600_O</td>\n",
       "      <td>9600_9600</td>\n",
       "      <td>9600_O_9600</td>\n",
       "      <td>1</td>\n",
       "      <td>142.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24861</th>\n",
       "      <td>f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda</td>\n",
       "      <td>340207</td>\n",
       "      <td>J</td>\n",
       "      <td>6790</td>\n",
       "      <td>4500</td>\n",
       "      <td>4530</td>\n",
       "      <td>6</td>\n",
       "      <td>340200000000000000</td>\n",
       "      <td>340200000000115797</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4533_J</td>\n",
       "      <td>4533_4500</td>\n",
       "      <td>4533_J_4500</td>\n",
       "      <td>0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24862</th>\n",
       "      <td>da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7</td>\n",
       "      <td>340207</td>\n",
       "      <td>O</td>\n",
       "      <td>8111</td>\n",
       "      <td>9600</td>\n",
       "      <td>无</td>\n",
       "      <td>6</td>\n",
       "      <td>340207030010000000</td>\n",
       "      <td>340200000000115275</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9600_O</td>\n",
       "      <td>9600_9600</td>\n",
       "      <td>9600_O_9600</td>\n",
       "      <td>1</td>\n",
       "      <td>103.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24863</th>\n",
       "      <td>516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31</td>\n",
       "      <td>340225</td>\n",
       "      <td>O</td>\n",
       "      <td>8090</td>\n",
       "      <td>1100</td>\n",
       "      <td>1130</td>\n",
       "      <td>7</td>\n",
       "      <td>340200000000116750</td>\n",
       "      <td>341400000000015220</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1130_O</td>\n",
       "      <td>1130_1100</td>\n",
       "      <td>1130_O_1100</td>\n",
       "      <td>1</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24864</th>\n",
       "      <td>9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a</td>\n",
       "      <td>340222</td>\n",
       "      <td>N</td>\n",
       "      <td>7830</td>\n",
       "      <td>9600</td>\n",
       "      <td>9600</td>\n",
       "      <td>6</td>\n",
       "      <td>340222070010000000</td>\n",
       "      <td>340200000020003395</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9600_N</td>\n",
       "      <td>9600_9600</td>\n",
       "      <td>9600_N_9600</td>\n",
       "      <td>1</td>\n",
       "      <td>115.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24865 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     id  oplocdistrict  \\\n",
       "0      47645761dc56bb8c5fae00114b768b5d9b6e917c3aec07c4         340223   \n",
       "1      9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3         340222   \n",
       "2      59b38c56de3836838082cfcb1a298951abfe15e6940c49ba         340202   \n",
       "3      e9f7b28ec10e047000d16ab79e1b5e6da434a1697cce7818         340221   \n",
       "4      f000950527a6feb63ee1ce82bb22ddd1ab8b8fdffa3b91fb         340202   \n",
       "...                                                 ...            ...   \n",
       "24860  f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325         340225   \n",
       "24861  f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda         340207   \n",
       "24862  da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7         340207   \n",
       "24863  516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31         340225   \n",
       "24864  9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a         340222   \n",
       "\n",
       "      industryphy industryco enttype enttypeitem  state               orgid  \\\n",
       "0               M       7513    1100        1150      6  340223010010000000   \n",
       "1               O       8090    9600           无      6  340222060010000000   \n",
       "2               R       9053    1100        1150      6  340202010010000000   \n",
       "3               L       7212    4500        4540      6  340221010010000000   \n",
       "4               R       8810    1100        1130      7  340200000000000000   \n",
       "...           ...        ...     ...         ...    ...                 ...   \n",
       "24860           O       8131    9600           无      6  340200000000116780   \n",
       "24861           J       6790    4500        4530      6  340200000000000000   \n",
       "24862           O       8111    9600           无      6  340207030010000000   \n",
       "24863           O       8090    1100        1130      7  340200000000116750   \n",
       "24864           N       7830    9600        9600      6  340222070010000000   \n",
       "\n",
       "                    jobid  adbusign  ...  enttypegb_industryphy  \\\n",
       "0      340200000000115392         0  ...                 1151_M   \n",
       "1      340200000000112114         0  ...                 9600_O   \n",
       "2      400000000000753910         0  ...                 1151_R   \n",
       "3      400000000000013538         0  ...                 4540_L   \n",
       "4      400000000000283237         0  ...                 1130_R   \n",
       "...                   ...       ...  ...                    ...   \n",
       "24860  341400000000011622         0  ...                 9600_O   \n",
       "24861  340200000000115797         0  ...                 4533_J   \n",
       "24862  340200000000115275         0  ...                 9600_O   \n",
       "24863  341400000000015220         0  ...                 1130_O   \n",
       "24864  340200000020003395         0  ...                 9600_N   \n",
       "\n",
       "       enttypegb_enttype  enttypegb_industryphy_enttype  null_enttypeminu  \\\n",
       "0              1151_1100                    1151_M_1100                 0   \n",
       "1              9600_9600                    9600_O_9600                 1   \n",
       "2              1151_1100                    1151_R_1100                 0   \n",
       "3              4540_4500                    4540_L_4500                 1   \n",
       "4              1130_1100                    1130_R_1100                 1   \n",
       "...                  ...                            ...               ...   \n",
       "24860          9600_9600                    9600_O_9600                 1   \n",
       "24861          4533_4500                    4533_J_4500                 0   \n",
       "24862          9600_9600                    9600_O_9600                 1   \n",
       "24863          1130_1100                    1130_O_1100                 1   \n",
       "24864          9600_9600                    9600_N_9600                 1   \n",
       "\n",
       "      alive_year  regcap_diff_reccap  regcap_div_reccap bucket_regcap  \\\n",
       "0           16.0                52.0              -0.04             2   \n",
       "1           38.0                12.0              -0.20             0   \n",
       "2            2.0               102.0              -0.02             4   \n",
       "3           62.0                12.0              -0.20             0   \n",
       "4           36.0               102.0              -0.02             4   \n",
       "...          ...                 ...                ...           ...   \n",
       "24860      142.0                22.0              -0.10             1   \n",
       "24861       59.0               110.0               0.00             5   \n",
       "24862      103.0                12.0              -0.20             0   \n",
       "24863       98.0                 0.0               1.00             5   \n",
       "24864      115.0                12.0              -0.20             0   \n",
       "\n",
       "       bucket_reccap  bucket_regcap_diff_reccap  \n",
       "0                  0                          3  \n",
       "1                  0                          1  \n",
       "2                  0                          5  \n",
       "3                  0                          1  \n",
       "4                  0                          5  \n",
       "...              ...                        ...  \n",
       "24860              0                          2  \n",
       "24861              0                          5  \n",
       "24862              0                          1  \n",
       "24863              2                          0  \n",
       "24864              0                          1  \n",
       "\n",
       "[24865 rows x 45 columns]"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_info_clean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- id:企业唯一标识\n",
    "- ANCHEYEAR:年度\n",
    "- STATE:状态\n",
    "- FUNDAM:资金数额\n",
    "- MEMNUM:成员人数\n",
    "- FARNUM:农民人数\n",
    "- ANNNEWMEMNUM:本年度新增成员人数\n",
    "- ANNREDMEMNUM:本年度退出成员人数\n",
    "- EMPNUM:从业人数\n",
    "- EMPNUMSIGN:从业人数是否公示\n",
    "- BUSSTNAME:经营状态名称\n",
    "- COLGRANUM:其中高校毕业生人数经营者\n",
    "- RETSOLNUM:其中退役士兵人数经营者\n",
    "- DISPERNUM:其中残疾人人数经营者\n",
    "- UNENUM:其中下岗失业人数经营者\n",
    "- COLEMPLNUM:其中高校毕业生人数雇员\n",
    "- RETEMPLNUM:其中退役士兵人数雇员\n",
    "- DISEMPLNUM:其中残疾人人数雇员\n",
    "- UNEEMPLNUM:其中下岗失业人数雇员\n",
    "- WEBSITSIGN:是否有网站标志\n",
    "- FORINVESTSIGN:是否有对外投资企业标志\n",
    "- STOCKTRANSIGN:有限责任公司本年度是否发生股东股权转让标志\n",
    "- PUBSTATE:公示状态：1 全部公示，2部分公示,3全部不公示]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 45940/45940 [00:00<00:00, 178598.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished .............\n"
     ]
    }
   ],
   "source": [
    "#change_info\n",
    "# [id:企业唯一标识, bgxmdm:变更信息代码, bgq:变更前, bgh:变更后, bgrq:变更日期]\n",
    "change_info_clean=change_info.copy()\n",
    "change_info_clean['bgrq']=pd.to_datetime(change_info_clean['bgrq'])\n",
    "current_time=pd.to_datetime('2020-11-17 00:00:00')\n",
    "#变更日期距今长度\n",
    "change_info_clean['bgrq_gap']=(current_time-pd.to_datetime(change_info_clean['bgrq'])).dt.total_seconds()//3600//24//30\n",
    "#将变更前、后的代码作一个交叉\n",
    "change_info_clean['bgq']=change_info_clean['bgq'].fillna(\"无\")\n",
    "change_info_clean['bgh']=change_info_clean['bgh'].fillna(\"无\")\n",
    "new_col=cross_two(change_info_clean,'bgq','bgh')#作企业类型-小类的交叉特征\n",
    "change_info_clean['bgq_bgh']=new_col\n",
    "#\n",
    "change_info_clean=change_info_clean.drop(['bgrq','bgq','bgh'],axis=1)\n",
    "change_info_clean=change_info_clean.groupby('id',sort=False).agg({\"bgxmdm\":[\"count\",\"sum\",\"mean\",\"std\"],\n",
    "                                             \"bgrq_gap\":[\"sum\",\"mean\",\"max\",\"min\"],\n",
    "                                             \"bgq_bgh\":[\"median\",\"mean\"],\n",
    "                                            }).reset_index()\n",
    "change_info_clean.columns=[\"id\",\"change_count\",\"bgxmdm_sum\",\"bgxmdm_mean\",\"bgxmdm_std\",\n",
    "                           \"bgrq_gap_sum\",\"bgrq_gap_mean\",\"bgrq_gap_max\",\"bgrq_gap_min\",\n",
    "                           \"bgq_bgh_median\",\"bgq_bgh_mean\"]\n",
    "#other_info\n",
    "#由于other信息缺失较多，所以对某一个id而言，将是否具有ohther信息作为一个特征\n",
    "other_info_clean = other_info.groupby('id',sort=False).agg('mean')\n",
    "other_info_clean=pd.DataFrame(other_info_clean).reset_index()\n",
    "other_info_clean=other_info_clean.fillna(-2)\n",
    "other_info_clean['has_other']=[1]*len(other_info_clean)\n",
    "#news_info\n",
    "#news_info\n",
    "#找到一些异常数据单独处理\n",
    "news_info_clean=news_info.copy()\n",
    "public_date_values=news_info_clean['public_date'].values\n",
    "for i in range(len(public_date_values)):\n",
    "    if len(public_date_values[i])!=10:\n",
    "        public_date_values[i]='2020-11-16'\n",
    "news_info_clean['public_date']=public_date_values\n",
    "news_info_clean['public_date']=pd.to_datetime(news_info_clean['public_date'])\n",
    "current_time=pd.to_datetime('2020-11-17 00:00:00')\n",
    "news_info_clean['news_gap']=(current_time-pd.to_datetime(news_info_clean['public_date'])).dt.total_seconds()//3600//24//30\n",
    "news_info_clean=news_info_clean.drop(['public_date'],axis=1)\n",
    "#对positive_negtive类型进行编码\n",
    "news_info_clean['positive_negtive']=news_info_clean['positive_negtive'].fillna(\"中立\")\n",
    "#\n",
    "dic={}\n",
    "cate=news_info_clean.positive_negtive.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "#\n",
    "news_info_clean['positive_negtive']=news_info_clean['positive_negtive'].map(dic)\n",
    "news_info_clean = news_info_clean.groupby('id',sort=False).agg({'positive_negtive':['mean','min','max'],\n",
    "                                                               'news_gap':['mean']}).reset_index()\n",
    "news_info_clean.columns=['id','positive_negtive_mean','positive_negtive_min','positive_negtive_max','news_gap_mean']\n",
    "\n",
    "#处理annual_report_info的数据\n",
    "#空值大于0.5的列都删除掉\n",
    "drop_nan_columns=get_nan_ratio(annual_report_info,thresh=0.5)[0]\n",
    "annual_report_info_clean=annual_report_info.drop(drop_nan_columns,axis=1)\n",
    "#对object类型进行编码\n",
    "annual_report_info_clean['BUSSTNAME']=annual_report_info_clean['BUSSTNAME'].fillna(\"无\")\n",
    "dic = {'无':-1,'开业':0, '歇业':1, '停业':2, '清算':3}\n",
    "annual_report_info_clean['BUSSTNAME']=annual_report_info_clean['BUSSTNAME'].map(dic)\n",
    "#\n",
    "annual_report_info_clean['sum_frt_1']=annual_report_info_clean['COLGRANUM']+annual_report_info_clean['RETSOLNUM']+annual_report_info_clean['DISPERNUM']+annual_report_info_clean['UNENUM']\n",
    "annual_report_info_clean['sum_frt_2']=annual_report_info_clean['COLEMPLNUM']+annual_report_info_clean['RETEMPLNUM']+annual_report_info_clean['DISEMPLNUM']+annual_report_info_clean['UNEEMPLNUM']\n",
    "annual_report_info_clean['ratio_1']=annual_report_info_clean['sum_frt_1']/annual_report_info_clean['EMPNUM']\n",
    "annual_report_info_clean['ratio_2']=annual_report_info_clean['sum_frt_2']/annual_report_info_clean['EMPNUM']\n",
    "annual_report_info_clean = annual_report_info_clean.groupby('id',sort=False).agg('median')\n",
    "annual_report_info_clean=pd.DataFrame(annual_report_info_clean).reset_index()\n",
    "tax_info_clean=tax_info.copy()\n",
    "tax_info_clean[\"has_tax\"]=[1]*len(tax_info_clean)\n",
    "tax_info_clean['START_DATE']=pd.to_datetime(tax_info_clean['START_DATE'])\n",
    "tax_info_clean['END_DATE']=pd.to_datetime(tax_info_clean['END_DATE'])\n",
    "tax_info_clean['gap_day']=(tax_info_clean['END_DATE']-tax_info_clean['START_DATE']).dt.total_seconds()//3600//24\n",
    "tax_info_clean=tax_info_clean.drop(['START_DATE','END_DATE'],axis=1)\n",
    "tax_info_clean['TAX_CATEGORIES']=tax_info_clean['TAX_CATEGORIES'].fillna(\"无\")#17 unique\n",
    "tax_info_clean['TAX_ITEMS']=tax_info_clean['TAX_ITEMS'].fillna(\"无\")#275 TAX_ITEMS\n",
    "#对object类型进行编码\n",
    "dic={}\n",
    "cate=tax_info_clean.TAX_CATEGORIES.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "tax_info_clean['TAX_CATEGORIES']=tax_info_clean['TAX_CATEGORIES'].map(dic)\n",
    "#\n",
    "dic={}\n",
    "cate=tax_info_clean.TAX_ITEMS.unique()\n",
    "for i in range(len(cate)):\n",
    "    dic[cate[i]]=i\n",
    "tax_info_clean['TAX_ITEMS']=tax_info_clean['TAX_ITEMS'].map(dic)\n",
    "tax_info_clean['income']=tax_info_clean['TAX_AMOUNT']/tax_info_clean['TAX_RATE']\n",
    "#\n",
    "tax_info_clean=tax_info_clean.groupby('id',sort=False).agg({\"TAX_CATEGORIES\":[\"count\",\"mean\",\"std\"],\n",
    "                                             \"TAX_ITEMS\":[\"mean\"],\n",
    "                                             \"TAXATION_BASIS\":[\"mean\"],\n",
    "                                             \"TAX_RATE\":[\"mean\"],\n",
    "                                             \"DEDUCTION\":[\"mean\"],\n",
    "                                             \"TAX_AMOUNT\":[\"sum\",\"mean\"],\n",
    "                                             \"gap_day\":[\"mean\"],\n",
    "                                             \"income\":[\"sum\",\"mean\"],\n",
    "                                             \"has_tax\":[\"mean\"]\n",
    "                                            }).reset_index()\n",
    "tax_info_clean.columns=['id','tax_count','TAX_CATEGORIES_mean','TAX_CATEGORIES_std','TAX_ITEMS','TAXATION_BASIS',\n",
    "                           'TAX_RATE','DEDUCTION','TAX_AMOUNT_sum','TAX_AMOUNT_mean','gap_day','income_sum','income_mean','has_tax']\n",
    "#税额分箱\n",
    "tax_info_clean['TAX_AMOUNT_sum']=tax_info_clean['TAX_AMOUNT_sum'].fillna(tax_info_clean['TAX_AMOUNT_sum'].median())\n",
    "tax_info_clean['bucket_TAX_AMOUNT']=pd.qcut(tax_info_clean['TAX_AMOUNT_sum'], 10, labels=False,duplicates='drop')\n",
    "tax_info_clean=tax_info_clean.fillna(-2)\n",
    "print('finished .............')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## category、object_column 特征单独提取出来\n",
    "\n",
    "为了避免在使用有监督编码过程中的特征穿越问题，不能笼统的针对所有的类别特征都采取同样的编码方式,\n",
    "\n",
    "需要结合其自身的特点进行编码;\n",
    "\n",
    "- 对于较为稀疏的cate列，类别数多，目标只有两类，因此很容易造成特征穿越。不适宜采取有监督编码。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all_data:(24865, 98) train_df:(14865, 99) test_df:(10000, 98)\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "all_data=base_info_clean.merge(annual_report_info_clean,how='outer')\n",
    "all_data=all_data.merge(tax_info_clean,how='outer')\n",
    "all_data=all_data.merge(change_info_clean,how='outer')\n",
    "all_data=all_data.merge(news_info_clean,how='outer')\n",
    "all_data=all_data.merge(other_info_clean,how='outer')\n",
    "all_data=all_data.fillna(-2)\n",
    "all_data.shape#,base_info.shape,annual_report_info.shape,tax_info.shape\n",
    "train_df=all_data.merge(entprise_info)\n",
    "test_df=all_data[all_data['id'].isin(entprise_evaluate['id'].unique().tolist())]\n",
    "test_df=test_df.reset_index(drop=True)\n",
    "print(\"all_data:{} train_df:{} test_df:{}\".format(all_data.shape,train_df.shape,test_df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对OBJ_FEATURES特征进行编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "OBJ_FEATURES=all_data.select_dtypes(include=[\"object\"]).columns.tolist()\n",
    "OBJ_FEATURES.remove('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "#编码之前先进行一个简单的统计(主要是针对特征类别的稀疏性)\n",
    "ordinal_encode=[]\n",
    "supervize_encode=[]\n",
    "for cat in OBJ_FEATURES:\n",
    "    if len(all_data)/len(all_data[cat].unique())<100:#>平均不足100个样本,因此很难表达其统计特性\n",
    "        ordinal_encode.append(cat)\n",
    "    else:\n",
    "        supervize_encode.append(cat)\n",
    "#\n",
    "encoder_tools=Category_encoders_hepler()\n",
    "#train_df,test_df=encoder_tools.encode(train_df,test_df,ordinal_encode,mode=\"ordinal\")\n",
    "#train_df,test_df=encoder_tools.encode(train_df,test_df,supervize_encode,mode=\"ordinal\")\n",
    "train_df,test_df=encoder_tools.encode(train_df,test_df,OBJ_FEATURES,mode=\"ordinal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((14865, 97), (10000, 97))"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "train_data=train_df.drop(['id','label'],axis=1)\n",
    "kind=train_df['label']\n",
    "test_data=test_df.drop(['id'],axis=1)\n",
    "train_data.shape,test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oplocdistrict</th>\n",
       "      <th>state</th>\n",
       "      <th>orgid</th>\n",
       "      <th>jobid</th>\n",
       "      <th>adbusign</th>\n",
       "      <th>townsign</th>\n",
       "      <th>regtype</th>\n",
       "      <th>empnum</th>\n",
       "      <th>compform</th>\n",
       "      <th>venind</th>\n",
       "      <th>...</th>\n",
       "      <th>enttype_enttypeitem_industryphy_industryco_ordinal</th>\n",
       "      <th>industryphy_id_prefix_ordinal</th>\n",
       "      <th>enttype_id_prefix_ordinal</th>\n",
       "      <th>industryphy_industryco_id_prefix_ordinal</th>\n",
       "      <th>enttype_enttypeitem_id_prefix_ordinal</th>\n",
       "      <th>industryphy_dom_prefix_ordinal</th>\n",
       "      <th>enttype_dom_prefix_ordinal</th>\n",
       "      <th>enttypegb_industryphy_ordinal</th>\n",
       "      <th>enttypegb_enttype_ordinal</th>\n",
       "      <th>enttypegb_industryphy_enttype_ordinal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>340223</td>\n",
       "      <td>6</td>\n",
       "      <td>340223010010000000</td>\n",
       "      <td>340200000000115392</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>340202</td>\n",
       "      <td>6</td>\n",
       "      <td>340202010010000000</td>\n",
       "      <td>400000000000753910</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>340221</td>\n",
       "      <td>6</td>\n",
       "      <td>340221010010000000</td>\n",
       "      <td>400000000000013538</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>340202</td>\n",
       "      <td>7</td>\n",
       "      <td>340200000000000000</td>\n",
       "      <td>400000000000283237</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>340222</td>\n",
       "      <td>6</td>\n",
       "      <td>340222080010000000</td>\n",
       "      <td>340200000000101006</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14860</th>\n",
       "      <td>340202</td>\n",
       "      <td>6</td>\n",
       "      <td>340202010010000000</td>\n",
       "      <td>340200000020001145</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14861</th>\n",
       "      <td>340200</td>\n",
       "      <td>6</td>\n",
       "      <td>340200000000000000</td>\n",
       "      <td>340200000000111582</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>146</td>\n",
       "      <td>64</td>\n",
       "      <td>4</td>\n",
       "      <td>234</td>\n",
       "      <td>4</td>\n",
       "      <td>125</td>\n",
       "      <td>22</td>\n",
       "      <td>46</td>\n",
       "      <td>3</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14862</th>\n",
       "      <td>340202</td>\n",
       "      <td>6</td>\n",
       "      <td>340202010010000000</td>\n",
       "      <td>400000000000753910</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>79</td>\n",
       "      <td>79</td>\n",
       "      <td>2</td>\n",
       "      <td>322</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>22</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14863</th>\n",
       "      <td>340202</td>\n",
       "      <td>6</td>\n",
       "      <td>340202010010000000</td>\n",
       "      <td>340200000000100093</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>98</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>129</td>\n",
       "      <td>4</td>\n",
       "      <td>35</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14864</th>\n",
       "      <td>340203</td>\n",
       "      <td>6</td>\n",
       "      <td>340203010010000000</td>\n",
       "      <td>400000000000705891</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>158</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14865 rows × 97 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       oplocdistrict  state               orgid               jobid  adbusign  \\\n",
       "0             340223      6  340223010010000000  340200000000115392         0   \n",
       "1             340202      6  340202010010000000  400000000000753910         0   \n",
       "2             340221      6  340221010010000000  400000000000013538         0   \n",
       "3             340202      7  340200000000000000  400000000000283237         0   \n",
       "4             340222      6  340222080010000000  340200000000101006         0   \n",
       "...              ...    ...                 ...                 ...       ...   \n",
       "14860         340202      6  340202010010000000  340200000020001145         0   \n",
       "14861         340200      6  340200000000000000  340200000000111582         0   \n",
       "14862         340202      6  340202010010000000  400000000000753910         0   \n",
       "14863         340202      6  340202010010000000  340200000000100093         0   \n",
       "14864         340203      6  340203010010000000  400000000000705891         0   \n",
       "\n",
       "       townsign  regtype  empnum  compform  venind  ...  \\\n",
       "0             0        1     5.0      -2.0    -2.0  ...   \n",
       "1             0        1     2.0      -2.0    -2.0  ...   \n",
       "2             1        1     2.0      -2.0    -2.0  ...   \n",
       "3             0        1    -2.0      -2.0    -2.0  ...   \n",
       "4             1        1     5.0       1.0     3.0  ...   \n",
       "...         ...      ...     ...       ...     ...  ...   \n",
       "14860         1        1     5.0      -2.0    -2.0  ...   \n",
       "14861         1        1    -2.0      -2.0    -2.0  ...   \n",
       "14862         0        1     1.0      -2.0    -2.0  ...   \n",
       "14863         1        1    -2.0      -2.0    -2.0  ...   \n",
       "14864         0        1     1.0      -2.0    -2.0  ...   \n",
       "\n",
       "       enttype_enttypeitem_industryphy_industryco_ordinal  \\\n",
       "0                                                      1    \n",
       "1                                                      2    \n",
       "2                                                      3    \n",
       "3                                                      4    \n",
       "4                                                      5    \n",
       "...                                                  ...    \n",
       "14860                                                 13    \n",
       "14861                                                146    \n",
       "14862                                                 79    \n",
       "14863                                                 98    \n",
       "14864                                                 21    \n",
       "\n",
       "       industryphy_id_prefix_ordinal  enttype_id_prefix_ordinal  \\\n",
       "0                                  1                          1   \n",
       "1                                  2                          2   \n",
       "2                                  3                          3   \n",
       "3                                  4                          4   \n",
       "4                                  5                          5   \n",
       "...                              ...                        ...   \n",
       "14860                             11                         11   \n",
       "14861                             64                          4   \n",
       "14862                             79                          2   \n",
       "14863                             12                          4   \n",
       "14864                              6                          6   \n",
       "\n",
       "       industryphy_industryco_id_prefix_ordinal  \\\n",
       "0                                             1   \n",
       "1                                             2   \n",
       "2                                             3   \n",
       "3                                             4   \n",
       "4                                             5   \n",
       "...                                         ...   \n",
       "14860                                        11   \n",
       "14861                                       234   \n",
       "14862                                       322   \n",
       "14863                                       129   \n",
       "14864                                       158   \n",
       "\n",
       "       enttype_enttypeitem_id_prefix_ordinal  industryphy_dom_prefix_ordinal  \\\n",
       "0                                          1                               1   \n",
       "1                                          2                               2   \n",
       "2                                          3                               3   \n",
       "3                                          4                               2   \n",
       "4                                          5                               4   \n",
       "...                                      ...                             ...   \n",
       "14860                                     12                               8   \n",
       "14861                                      4                             125   \n",
       "14862                                      2                              20   \n",
       "14863                                      4                              35   \n",
       "14864                                     22                               1   \n",
       "\n",
       "       enttype_dom_prefix_ordinal  enttypegb_industryphy_ordinal  \\\n",
       "0                               1                              1   \n",
       "1                               1                              2   \n",
       "2                               2                              3   \n",
       "3                               1                              4   \n",
       "4                               3                              5   \n",
       "...                           ...                            ...   \n",
       "14860                           6                             11   \n",
       "14861                          22                             46   \n",
       "14862                          22                             15   \n",
       "14863                          22                              7   \n",
       "14864                           1                              1   \n",
       "\n",
       "       enttypegb_enttype_ordinal  enttypegb_industryphy_enttype_ordinal  \n",
       "0                              1                                      1  \n",
       "1                              1                                      2  \n",
       "2                              2                                      3  \n",
       "3                              3                                      4  \n",
       "4                              4                                      5  \n",
       "...                          ...                                    ...  \n",
       "14860                          8                                     11  \n",
       "14861                          3                                     46  \n",
       "14862                          1                                     15  \n",
       "14863                          3                                      7  \n",
       "14864                          1                                      1  \n",
       "\n",
       "[14865 rows x 97 columns]"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 伪标签\n",
    "# use_pseudo=True\n",
    "# if use_pseudo:\n",
    "#     train_data=train_df.drop(['id','label'],axis=1)\n",
    "#     kind=train_df['label']\n",
    "#     pseudo_name=[]\n",
    "#     pseudo_label={'id':[],'label':[]}\n",
    "#     pseudo_df=pd.read_csv('submit_857_xgb_rf_lgb_cab.csv')\n",
    "#     for index,name,score in pseudo_df.itertuples():\n",
    "#         if score>0.9 or score<0.05:\n",
    "#             pseudo_label['id'].append(name)\n",
    "#             if score>0.9 :\n",
    "#                 pseudo_label['label'].append(1)\n",
    "#             else:\n",
    "#                 pseudo_label['label'].append(0)\n",
    "#             pseudo_name.append(name)\n",
    "#     len(pseudo_name)\n",
    "#     pseudo_data=test_df[test_df.id.isin(pseudo_name)].reset_index(drop=True)\n",
    "#     pseudo_data=pseudo_data.merge(pd.DataFrame(pseudo_label))\n",
    "#     #\n",
    "#     train_df=pd.concat((train_df,pseudo_data)).reset_index(drop=True)\n",
    "#     train_data=train_df.drop(['id','label'],axis=1)\n",
    "#     kind=train_df['label']\n",
    "#     print(train_data.shape,test_data.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "#特征筛选\n",
    "# frt_select=[\n",
    "#  'industryphy',\n",
    "#  'enttypegb',\n",
    "#  'regcap',\n",
    "#  'townsign',\n",
    "#  'industryco',\n",
    "#  'bucket_regcap',\n",
    "#  'empnum',\n",
    "#  'bucket_reccap',\n",
    "#  'enttypeitem',\n",
    "#  'industryphy_industryco',\n",
    "#  'reccap',\n",
    "#  'FORINVESTSIGN',\n",
    "#  'positive_negtive',\n",
    "#  'regtype',\n",
    "#  'STOCKTRANSIGN',\n",
    "#  'bucket_regcap_reccap',\n",
    "#  'enttypegb_enttypeitem',\n",
    "#  'regcap_reccap',\n",
    "#  'legal_judgment_num',\n",
    "#  'TAX_CATEGORIES',\n",
    "#  'TAX_AMOUNT',\n",
    "#  'bgq_bgh',\n",
    "#  'TAX_ITEMS']\n",
    "# frt_select=important_frt[:30]\n",
    "# train_data=train_data[frt_select]\n",
    "# test_data=test_data[frt_select]\n",
    "# cat_features=list(set(frt_select).intersection(set(cat_features)))\n",
    "# cat_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def eval_score(y_test,y_pre):\n",
    "#     _,_,f_class,_=precision_recall_fscore_support(y_true=y_test,y_pred=y_pre,labels=[0,1],average=None)\n",
    "#     fper_class={'合法':f_class[0],'违法':f_class[1],'f1':f1_score(y_test,y_pre)}\n",
    "#     return fper_class\n",
    "#\n",
    "def eval_score(y_test,y_pre):\n",
    "    valid_f1 = f1_score(y_test, y_pre)\n",
    "    valid_p = precision_score(y_test, y_pre)\n",
    "    valid_r = recall_score(y_test, y_pre)\n",
    "    F = valid_p*0.7+valid_r*0.2+valid_f1*0.1\n",
    "    return F\n",
    "\n",
    "#\n",
    "def k_fold_serachParmaters(model,train_val_data,train_val_kind):\n",
    "    mean_f1=0\n",
    "    mean_f1Train=0\n",
    "    n_splits=5\n",
    "    sk = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2020)\n",
    "    for train, test in sk.split(train_val_data, train_val_kind):\n",
    "        x_train = train_val_data.iloc[train]\n",
    "        y_train = train_val_kind.iloc[train]\n",
    "        x_test = train_val_data.iloc[test]\n",
    "        y_test = train_val_kind.iloc[test]\n",
    "\n",
    "        model.fit(x_train, y_train)\n",
    "        pred = model.predict(x_test)\n",
    "        weigted_score=  eval_score(y_test,pred)\n",
    "        mean_f1+=weigted_score/n_splits\n",
    "        #print(fper_class)\n",
    "        \n",
    "        pred_Train = model.predict(x_train)\n",
    "        weigted_score_train =  eval_score(y_train,pred_Train)\n",
    "        mean_f1Train+=weigted_score_train/n_splits\n",
    "    #print('mean valf1:',mean_f1)\n",
    "    #print('mean trainf1:',mean_f1Train)\n",
    "    return mean_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 150\n",
      "[150, 0.05, 7, 7] 0.8308745056274318\n",
      "[150, 0.05, 7, 11] 0.8314844234174977\n",
      "[150, 0.05, 8, 11] 0.8319370224286234\n",
      "[150, 0.06, 7, 13] 0.8338354962736693\n",
      "[150, 0.06, 13, 13] 0.8341930790252486\n",
      "[150, 0.07, 13, 11] 0.8370422016216938\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,num_leaves):\n",
    "    params={'num_leaves':num_leaves\n",
    "                   ,'max_depth':max_depth\n",
    "                   ,'learning_rate':lr\n",
    "                   ,'n_estimators':n_estimators\n",
    "                   ,'n_jobs':8\n",
    "               }\n",
    "    mean_f1=k_fold_serachParmaters(lightgbm.LGBMClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [150]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.05,0.06,0.07,0.1]:\n",
    "        for max_depth in [7,8,9,13,11]:\n",
    "            for num_leaves in [7,9,11,13]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,num_leaves)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,num_leaves]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[150, 0.07, 13, 11] 0.8370422016216938"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 80\n",
      "[80, 0.05, 7, 7] 0.8256217607168728\n",
      "[80, 0.05, 7, 9] 0.8284444493130954\n",
      "[80, 0.05, 10, 9] 0.829558150242842\n",
      "[80, 0.07, 7, 7] 0.8299533354880131\n",
      "[80, 0.07, 10, 9] 0.8304676811777514\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "def search_param(n_estimators,lr,max_depth,min_child_weight):\n",
    "    params={'max_depth':max_depth\n",
    "              ,'learning_rate':lr\n",
    "              ,'n_estimators':n_estimators\n",
    "              ,'min_child_weight':min_child_weight\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "           }\n",
    "    \n",
    "    mean_f1=k_fold_serachParmaters(xgboost.XGBClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [80]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.05,0.07,0.09,0.1]:\n",
    "        for max_depth in [7,8,9,10]:\n",
    "            for min_child_weight in [7,9,11,13,15]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,min_child_weight)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,min_child_weight]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[50, 0.04, 4, 13] 0.8449774193716448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter_cnt: 100\n",
      "[100, 0.08, 8] 0.8385293441728544\n",
      "[100, 0.08, 8] 0.8385293441728544\n"
     ]
    }
   ],
   "source": [
    "def search_param(iter_cnt,lr,max_depth):\n",
    "    \n",
    "    params={'iterations':iter_cnt\n",
    "              ,'learning_rate':lr\n",
    "              ,'depth':max_depth\n",
    "              ,'silent':True\n",
    "              ,'thread_count':8\n",
    "              ,'task_type':'CPU'\n",
    "        }\n",
    "    mean_f1=k_fold_serachParmaters(catboost.CatBoostClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数[52, 0.073, 7] 0.8440488273317126\n",
    "param=[]\n",
    "best=0\n",
    "for iter_cnt in [100]:\n",
    "    print('iter_cnt:',iter_cnt)\n",
    "    for lr in [0.08,0.1]:\n",
    "        for max_depth in [8]:\n",
    "            mean_f1=search_param(iter_cnt,lr,max_depth)\n",
    "            if mean_f1>best:\n",
    "                param=[iter_cnt,lr,max_depth]#[55, 0.076, 7]\n",
    "                best=mean_f1\n",
    "                print(param,best)\n",
    "print(param,best)#筛选特征:[70, 0.06 8] 0.8417684642475657 所有特征:[54, 0.07, 7] 0.8411337269934891\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 60\n",
      "[60, 7, 8] 0.8124315055365392\n",
      "[60, 9, 8] 0.832002118438721\n",
      "[60, 11, 8] 0.8325276971301041\n",
      "[60, 11, 9] 0.8356076453263122\n"
     ]
    }
   ],
   "source": [
    "# #\n",
    "def search_param(n_estimators,max_depth,min_samples_split):\n",
    "    params={'oob_score':True,\n",
    "            'random_state':2020,\n",
    "            'n_estimators': n_estimators,\n",
    "            'max_depth':max_depth,\n",
    "            'min_samples_split':min_samples_split\n",
    "            }\n",
    "    mean_f1=k_fold_serachParmaters(RandomForestClassifier(**params),train_data,kind)\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [60]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for min_samples_split in [8,9]:\n",
    "        for max_depth in [7,9,11]:\n",
    "            mean_f1=search_param(n_estimators,max_depth,min_samples_split)\n",
    "            if mean_f1>best:\n",
    "                param=[n_estimators,max_depth,min_samples_split]\n",
    "                best=mean_f1\n",
    "                print(param,best)#[60, 11, 9] 0.8442795786278773\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier_v1: 0.8685179961087608\n",
      "RandomForestClassifier_v2: 0.8356076453263122\n",
      "ExtraTreesClassifier: 0.8330043205908855\n",
      "GradientBoostingClassifier: 0.8339789173746215\n",
      "LGBMClassifier: 0.8370422016216938\n",
      "XGBClassifier: 0.8304676811777514\n",
      "CatBoostClassifier: 0.8355527942567414\n"
     ]
    }
   ],
   "source": [
    "rf_v1_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators': 50,\n",
    "     'warm_start': True, \n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 8,\n",
    "    'min_samples_leaf': 2,\n",
    "    'max_features' : 'sqrt',\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "rf_v2_params = {\n",
    "    'oob_score':True,\n",
    "    'random_state':2020,\n",
    "    'n_estimators': 60,\n",
    "    'max_depth':11,\n",
    "    'min_samples_split':9\n",
    "}\n",
    "# Extra Trees Parameters\n",
    "et_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators':60,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 20,\n",
    "    'min_samples_leaf': 1,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "\n",
    "\n",
    "# Gradient Boosting parameters\n",
    "gb_params = {\n",
    "    'n_estimators': 60,\n",
    "    'max_features': 0.2,\n",
    "    'max_depth': 5,\n",
    "    'min_samples_leaf': 2,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "#\n",
    "lgb_params = {'num_leaves':11\n",
    "                ,'max_depth':13\n",
    "                ,'learning_rate':0.07\n",
    "                ,'n_estimators':150\n",
    "                ,'n_jobs':8\n",
    "             }\n",
    "xgb_params ={'max_depth':10\n",
    "              ,'learning_rate':0.07\n",
    "              ,'n_estimators':80\n",
    "             ,'min_child_weight':9\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "            }\n",
    "cab_params={'iterations':70\n",
    "          ,'learning_rate':0.06\n",
    "          ,'depth':8\n",
    "          ,'silent':True\n",
    "          ,'thread_count':8\n",
    "          ,'task_type':'CPU'\n",
    "         # ,'cat_features':CAT_FEATURES_INT\n",
    "}\n",
    "\n",
    "rf_v1 = RandomForestClassifier(**rf_v1_params)\n",
    "print(\"RandomForestClassifier_v1:\",k_fold_serachParmaters(rf_v1,train_data,kind))\n",
    "rf_v2 = RandomForestClassifier(**rf_v2_params)\n",
    "print(\"RandomForestClassifier_v2:\",k_fold_serachParmaters(rf_v2,train_data,kind))\n",
    "et = ExtraTreesClassifier(**et_params)\n",
    "print(\"ExtraTreesClassifier:\",k_fold_serachParmaters(et,train_data,kind))\n",
    "gb = GradientBoostingClassifier(**gb_params)\n",
    "print(\"GradientBoostingClassifier:\",k_fold_serachParmaters(gb,train_data,kind))\n",
    "lgb=lightgbm.LGBMClassifier(**lgb_params)\n",
    "print('LGBMClassifier:',k_fold_serachParmaters(lgb,train_data,kind))\n",
    "xgb=xgboost.XGBClassifier(**xgb_params)\n",
    "print('XGBClassifier:',k_fold_serachParmaters(xgb,train_data,kind))\n",
    "cab=catboost.CatBoostClassifier(**cab_params)\n",
    "print('CatBoostClassifier:',k_fold_serachParmaters(cab,train_data,kind))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier_v1: 0.8725420817706592\n",
      "RandomForestClassifier_v2: 0.8195476881154073\n",
      "ExtraTreesClassifier: 0.8143537162924571\n",
      "GradientBoostingClassifier: 0.8187633045803323\n",
      "LGBMClassifier: 0.8166846627861111\n",
      "XGBClassifier: 0.821511379363949\n",
      "CatBoostClassifier: 0.8250984560890233\n"
     ]
    }
   ],
   "source": [
    "rf_v1_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators': 50,\n",
    "     'warm_start': True, \n",
    "     #'max_features': 0.2,\n",
    "    'max_depth': 8,\n",
    "    'min_samples_leaf': 2,\n",
    "    'max_features' : 'sqrt',\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "rf_v2_params = {\n",
    "    'oob_score':True,\n",
    "    'random_state':2020,\n",
    "    'n_estimators': 60,\n",
    "    'max_depth':11,\n",
    "    'min_samples_split':9\n",
    "}\n",
    "# Extra Trees Parameters\n",
    "et_params = {\n",
    "    'n_jobs': -1,\n",
    "    'n_estimators':60,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 20,\n",
    "    'min_samples_leaf': 1,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "\n",
    "\n",
    "# Gradient Boosting parameters\n",
    "gb_params = {\n",
    "    'n_estimators': 60,\n",
    "    'max_features': 0.2,\n",
    "    'max_depth': 5,\n",
    "    'min_samples_leaf': 2,\n",
    "    'verbose': 0,\n",
    "    'random_state':2020,\n",
    "}\n",
    "#\n",
    "lgb_params = {'num_leaves':11\n",
    "                ,'max_depth':13\n",
    "                ,'learning_rate':0.07\n",
    "                ,'n_estimators':150\n",
    "                ,'n_jobs':8\n",
    "             }\n",
    "xgb_params ={'max_depth':10\n",
    "              ,'learning_rate':0.07\n",
    "              ,'n_estimators':80\n",
    "             ,'min_child_weight':9\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "            }\n",
    "cab_params={'iterations':70\n",
    "          ,'learning_rate':0.06\n",
    "          ,'depth':8\n",
    "          ,'silent':True\n",
    "          ,'thread_count':8\n",
    "          ,'task_type':'CPU'\n",
    "         # ,'cat_features':CAT_FEATURES_INT\n",
    "}\n",
    "\n",
    "rf_v1 = RandomForestClassifier(**rf_v1_params)\n",
    "print(\"RandomForestClassifier_v1:\",k_fold_serachParmaters(rf_v1,train_data,kind))\n",
    "rf_v2 = RandomForestClassifier(**rf_v2_params)\n",
    "print(\"RandomForestClassifier_v2:\",k_fold_serachParmaters(rf_v2,train_data,kind))\n",
    "et = ExtraTreesClassifier(**et_params)\n",
    "print(\"ExtraTreesClassifier:\",k_fold_serachParmaters(et,train_data,kind))\n",
    "gb = GradientBoostingClassifier(**gb_params)\n",
    "print(\"GradientBoostingClassifier:\",k_fold_serachParmaters(gb,train_data,kind))\n",
    "lgb=lightgbm.LGBMClassifier(**lgb_params)\n",
    "print('LGBMClassifier:',k_fold_serachParmaters(lgb,train_data,kind))\n",
    "xgb=xgboost.XGBClassifier(**xgb_params)\n",
    "print('XGBClassifier:',k_fold_serachParmaters(xgb,train_data,kind))\n",
    "cab=catboost.CatBoostClassifier(**cab_params)\n",
    "print('CatBoostClassifier:',k_fold_serachParmaters(cab,train_data,kind))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "每1次验证的f1:0.8518143385936884\n",
      "每2次验证的f1:0.8411379589995327\n",
      "每3次验证的f1:0.8265306122448979\n",
      "每4次验证的f1:0.8427443301684486\n",
      "每5次验证的f1:0.8354570254497738\n",
      "mean f1: 0.8395368530912684\n"
     ]
    }
   ],
   "source": [
    "# #\n",
    "details = []\n",
    "answers = []\n",
    "mean_f1=0\n",
    "n_splits=5\n",
    "sk = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2020)\n",
    "cnt=0\n",
    "rf=rf_v2\n",
    "for train, test in sk.split(train_data, kind):\n",
    "    x_train = train_data.iloc[train]\n",
    "    y_train = kind.iloc[train]\n",
    "    x_test = train_data.iloc[test]\n",
    "    y_test = kind.iloc[test]\n",
    "\n",
    "    xgb.fit(x_train, y_train)\n",
    "    pred_xgb = xgb.predict(x_test)\n",
    "    weight_xgb = eval_score(y_test,pred_xgb)\n",
    "\n",
    "    lgb.fit(x_train, y_train)\n",
    "    pred_llf = lgb.predict(x_test)\n",
    "    weight_lgb = eval_score(y_test,pred_llf)\n",
    "\n",
    "    cab.fit(x_train, y_train)\n",
    "    pred_cab = cab.predict(x_test)\n",
    "    weight_cab =  eval_score(y_test,pred_cab)\n",
    "\n",
    "    rf.fit(x_train, y_train)\n",
    "    pred_rf = rf.predict(x_test)\n",
    "    weight_rf =  eval_score(y_test,pred_rf)\n",
    "\n",
    "\n",
    "    prob_xgb = xgb.predict_proba(x_test)\n",
    "    prob_lgb = lgb.predict_proba(x_test)\n",
    "    prob_cab = cab.predict_proba(x_test)\n",
    "    prob_rf = rf.predict_proba(x_test)\n",
    "    #print(prob_xgb.shape,prob_lgb.shape,prob_cab.shape,prob_rf.shape,)\n",
    "\n",
    "    scores = []\n",
    "    ijkl = []\n",
    "    weight = np.arange(0, 1.05, 0.1)\n",
    "    for i, item1 in enumerate(weight):\n",
    "        for j, item2 in enumerate(weight[weight <= (1 - item1)]):\n",
    "            for k, item3 in enumerate(weight[weight <= (1 - item1-item2)]):\n",
    "                prob_end = prob_xgb * item1 + prob_lgb * item2 + prob_cab *item3+prob_rf*(1 - item1 - item2-item3)\n",
    "                #prob_end = np.sqrt(prob_xgb**2 * item1 + prob_lgb**2 * item2 + prob_cab**2 *item3+prob_rf**2*(1 - item1 - item2-item3))\n",
    "                score = eval_score(y_test,np.argmax(prob_end,axis=1))\n",
    "                scores.append(score)\n",
    "                ijkl.append((item1, item2,item3, 1 - item1 - item2-item3))\n",
    "\n",
    "    ii = ijkl[np.argmax(scores)][0]\n",
    "    jj = ijkl[np.argmax(scores)][1]\n",
    "    kk = ijkl[np.argmax(scores)][2]\n",
    "    ll = ijkl[np.argmax(scores)][3]\n",
    "\n",
    "    details.append(max(scores))\n",
    "    details.append(weight_xgb)\n",
    "    details.append(weight_lgb)\n",
    "    details.append(weight_cab)\n",
    "    details.append(weight_rf)\n",
    "    details.append(ii)\n",
    "    details.append(jj)\n",
    "    details.append(kk)\n",
    "    details.append(ll)\n",
    "\n",
    "    cnt+=1\n",
    "    print('每{}次验证的f1:{}'.format(cnt,max(scores)))\n",
    "    mean_f1+=max(scores)/n_splits\n",
    "\n",
    "    test_xgb = xgb.predict_proba(test_data)\n",
    "    test_lgb = lgb.predict_proba(test_data)\n",
    "    test_cab = cab.predict_proba(test_data)\n",
    "    test_rf = rf.predict_proba(test_data)\n",
    "    #加权平均\n",
    "    ans = test_xgb * ii + test_lgb * jj + test_cab * kk + test_rf*ll#加权平均\n",
    "    #加权平方平均\n",
    "    #ans = np.sqrt(test_xgb**2 * ii + test_lgb**2 * jj + test_cab**2 * kk + test_rf**2*ll)\n",
    "    answers.append(ans)\n",
    "print('mean f1:',mean_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_end_score</th>\n",
       "      <th>xgboost</th>\n",
       "      <th>lightgbm</th>\n",
       "      <th>catboost</th>\n",
       "      <th>rf</th>\n",
       "      <th>weight_xgboost</th>\n",
       "      <th>weight_lightgbm</th>\n",
       "      <th>weight_catboost</th>\n",
       "      <th>weight_rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.851814</td>\n",
       "      <td>0.838174</td>\n",
       "      <td>0.824277</td>\n",
       "      <td>0.824999</td>\n",
       "      <td>0.844433</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.841138</td>\n",
       "      <td>0.838029</td>\n",
       "      <td>0.819034</td>\n",
       "      <td>0.831633</td>\n",
       "      <td>0.810354</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.826531</td>\n",
       "      <td>0.812136</td>\n",
       "      <td>0.812092</td>\n",
       "      <td>0.810755</td>\n",
       "      <td>0.795724</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.842744</td>\n",
       "      <td>0.825195</td>\n",
       "      <td>0.831633</td>\n",
       "      <td>0.822648</td>\n",
       "      <td>0.823163</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.835457</td>\n",
       "      <td>0.794022</td>\n",
       "      <td>0.796389</td>\n",
       "      <td>0.835457</td>\n",
       "      <td>0.824065</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test_end_score   xgboost  lightgbm  catboost        rf  weight_xgboost  \\\n",
       "0        0.851814  0.838174  0.824277  0.824999  0.844433             0.0   \n",
       "1        0.841138  0.838029  0.819034  0.831633  0.810354             0.6   \n",
       "2        0.826531  0.812136  0.812092  0.810755  0.795724             0.7   \n",
       "3        0.842744  0.825195  0.831633  0.822648  0.823163             0.2   \n",
       "4        0.835457  0.794022  0.796389  0.835457  0.824065             0.0   \n",
       "\n",
       "   weight_lightgbm  weight_catboost  weight_rf  \n",
       "0              0.3              0.2        0.5  \n",
       "1              0.1              0.2        0.1  \n",
       "2              0.1              0.1        0.1  \n",
       "3              0.3              0.3        0.2  \n",
       "4              0.0              1.0        0.0  "
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.DataFrame(np.array(details).reshape(int(len(details)/9),9)\n",
    "                ,columns=['test_end_score','xgboost','lightgbm','catboost','rf'\n",
    "                ,'weight_xgboost','weight_lightgbm','weight_catboost','weight_rf'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_end_score     0.839537\n",
       "xgboost            0.821511\n",
       "lightgbm           0.816685\n",
       "catboost           0.825098\n",
       "rf                 0.819548\n",
       "weight_xgboost     0.300000\n",
       "weight_lightgbm    0.160000\n",
       "weight_catboost    0.360000\n",
       "weight_rf          0.180000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../submission/submit_839.csv'"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "fina=sum(answers)/n_splits#\n",
    "#fina=np.sqrt(sum(np.array(answers)**2)/n_splits)#平方平均\n",
    "fina=fina[:,1]\n",
    "test_df['score']=fina#可选:fina_persudo是伪标签的预测结果\n",
    "submit_csv=test_df[['id','score']]\n",
    "save_path='../submission/'+'submit_'+str(int(mean_f1*1000))+'.csv'\n",
    "submit_csv.to_csv(save_path,index=False)\n",
    "save_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3</td>\n",
       "      <td>0.001898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>da8691b210adb3f67820f5e0c87b337d63112cee52211888</td>\n",
       "      <td>0.004178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17</td>\n",
       "      <td>0.001383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db</td>\n",
       "      <td>0.003688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f000950527a6feb617e8d6ca7025dcf9d765429969122069</td>\n",
       "      <td>0.004439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325</td>\n",
       "      <td>0.012233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda</td>\n",
       "      <td>0.453708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7</td>\n",
       "      <td>0.012285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31</td>\n",
       "      <td>0.039807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a</td>\n",
       "      <td>0.190291</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    id     score\n",
       "0     9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3  0.001898\n",
       "1     da8691b210adb3f67820f5e0c87b337d63112cee52211888  0.004178\n",
       "2     9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17  0.001383\n",
       "3     f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db  0.003688\n",
       "4     f000950527a6feb617e8d6ca7025dcf9d765429969122069  0.004439\n",
       "...                                                ...       ...\n",
       "9995  f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325  0.012233\n",
       "9996  f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda  0.453708\n",
       "9997  da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7  0.012285\n",
       "9998  516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31  0.039807\n",
       "9999  9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a  0.190291\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "预测结果中合法的数量9090;违法的数量910,合法/违法9.989011\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "def cnt_result(xx):\n",
    "    cnt_re={0:0,1:0}\n",
    "    for a in xx:\n",
    "        if a<=0.5:\n",
    "            cnt_re[0]+=1\n",
    "        else:\n",
    "            cnt_re[1]+=1\n",
    "    return cnt_re\n",
    "cnt_re=cnt_result(fina)\n",
    "print(\"预测结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#观察训练/验证过程\n",
    "# df=pd.DataFrame(np.array(details).reshape(int(len(details)/7),7)\n",
    "#                 ,columns=['test_end_score','xgboost','lightgbm','catboost'\n",
    "#                 ,'weight_xgboost','weight_lightgbm','weight_catboost'])\n",
    "# df\n",
    "\n",
    "# df.mean()\n",
    "\n",
    "xlf_impt=xgb.feature_importances_\n",
    "llf_impt=lgb.feature_importances_/sum(lgb.feature_importances_)\n",
    "clf_impt=cab.feature_importances_/sum(cab.feature_importances_)\n",
    "rf_impt=rf.feature_importances_/sum(rf.feature_importances_)\n",
    "\n",
    "importance=pd.DataFrame({\n",
    "    'column':train_data.columns,\n",
    "    'importance':xlf_impt+llf_impt+clf_impt+rf_impt,\n",
    "}).sort_values(by='importance')\n",
    "importance=importance.reset_index(drop=True)\n",
    "important_frt=list(importance['column'].values)\n",
    "important_frt.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alive_year',\n",
       " 'enttypegb_industryphy_ordinal',\n",
       " 'industryphy_enttype_ordinal',\n",
       " 'enttypeitem_ordinal',\n",
       " 'tfidif_opscope',\n",
       " 'industryphy_dom_prefix_ordinal',\n",
       " 'enttypegb_enttype_ordinal',\n",
       " 'industryphy_ordinal',\n",
       " 'regcap',\n",
       " 'enttype_enttypeitem_ordinal',\n",
       " 'enttypeminu',\n",
       " 'regcap_div_reccap',\n",
       " 'enttypegb',\n",
       " 'news_gap_mean',\n",
       " 'reccap',\n",
       " 'enttypegb_industryphy_enttype_ordinal',\n",
       " 'regcap_diff_reccap',\n",
       " 'industryphy_industryco_id_prefix_ordinal',\n",
       " 'bucket_regcap',\n",
       " 'enttype_enttypeitem_industryphy_industryco_ordinal',\n",
       " 'id_dom_ordinal',\n",
       " 'industryphy_id_prefix_ordinal',\n",
       " 'industryco_ordinal',\n",
       " 'enttype_ordinal',\n",
       " 'townsign',\n",
       " 'oplocdistrict',\n",
       " 'EMPNUM',\n",
       " 'jobid',\n",
       " 'empnum',\n",
       " 'enttype_enttypeitem_id_prefix_ordinal',\n",
       " 'bgxmdm_std',\n",
       " 'bgq_bgh_median',\n",
       " 'bgq_bgh_mean',\n",
       " 'enttype_dom_prefix_ordinal',\n",
       " 'industryphy_industryco_ordinal',\n",
       " 'bgxmdm_mean',\n",
       " 'dom_prefix_ordinal',\n",
       " 'bucket_regcap_diff_reccap',\n",
       " 'bgxmdm_sum',\n",
       " 'bucket_reccap',\n",
       " 'COLGRANUM',\n",
       " 'enttype_id_prefix_ordinal',\n",
       " 'FORINVESTSIGN',\n",
       " 'orgid',\n",
       " 'EMPNUMSIGN',\n",
       " 'change_count',\n",
       " 'opform_ordinal',\n",
       " 'legal_judgment_num',\n",
       " 'ANCHEYEAR',\n",
       " 'bgrq_gap_min',\n",
       " 'positive_negtive_mean',\n",
       " 'id_prefix_ordinal',\n",
       " 'sum_frt_2',\n",
       " 'ratio_1',\n",
       " 'ratio_2',\n",
       " 'PUBSTATE',\n",
       " 'venind',\n",
       " 'positive_negtive_max',\n",
       " 'STOCKTRANSIGN',\n",
       " 'TAX_ITEMS',\n",
       " 'compform',\n",
       " 'bgrq_gap_sum',\n",
       " 'positive_negtive_min',\n",
       " 'RETEMPLNUM',\n",
       " 'gap_day',\n",
       " 'sum_frt_1',\n",
       " 'COLEMPLNUM',\n",
       " 'TAX_AMOUNT_mean',\n",
       " 'state',\n",
       " 'WEBSITSIGN',\n",
       " 'TAX_CATEGORIES_std',\n",
       " 'has_other',\n",
       " 'regtype',\n",
       " 'TAX_CATEGORIES_mean',\n",
       " 'TAX_RATE',\n",
       " 'TAX_AMOUNT_sum',\n",
       " 'brand_num',\n",
       " 'null_enttypeminu',\n",
       " 'oploc_ordinal',\n",
       " 'UNEEMPLNUM',\n",
       " 'BUSSTNAME',\n",
       " 'has_tax',\n",
       " 'UNENUM',\n",
       " 'RETSOLNUM',\n",
       " 'tax_count',\n",
       " 'DISEMPLNUM',\n",
       " 'patent_num',\n",
       " 'DISPERNUM',\n",
       " 'income_sum',\n",
       " 'bgrq_gap_mean',\n",
       " 'bucket_TAX_AMOUNT',\n",
       " 'bgrq_gap_max',\n",
       " 'DEDUCTION',\n",
       " 'STATE',\n",
       " 'income_mean',\n",
       " 'TAXATION_BASIS',\n",
       " 'adbusign']"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "important_frt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 两层stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------->>>>>定义一个Sklearn classifier 的扩展<<<<<--------\n",
      "-------->>>>>定义oof:stacking的核心流程<<<<<--------\n",
      "-------->>>>>第一层的基模型<<<<<--------\n",
      "-------->>>>>训练、测试数据<<<<<--------\n",
      "y_train:(14865,);train_data:(14865, 97);test_data:(10000, 97)\n",
      "-------->>>>>第一阶段训练oof<<<<<--------\n",
      "Extra Trees.............\n",
      "Random Forest.............\n",
      "Gradient.............\n",
      "LGBClassifier.............\n",
      "XGBClassifier.............\n",
      "CatClassifier.............\n",
      "Training Stage_1 is complete\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "use_selectFrt=False\n",
    "ntrain = train_data.shape[0]\n",
    "ntest = test_data.shape[0]\n",
    "SEED = 0 # for reproducibility\n",
    "NFOLDS = 5 # set folds for out-of-fold prediction\n",
    "#kf = KFold(n_splits= NFOLDS, random_state=SEED)\n",
    "kf=StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=2020)\n",
    "print(\"-------->>>>>定义一个Sklearn classifier 的扩展<<<<<--------\")\n",
    "# Class to extend the Sklearn classifier\n",
    "class SklearnHelper(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "    \n",
    "    def predict_proba(self, x):\n",
    "        return self.clf.predict_proba(x)[:,1]\n",
    "    \n",
    "    def fit(self,x,y):\n",
    "        return self.clf.fit(x,y)\n",
    "    \n",
    "    def feature_importances(self,x,y):\n",
    "        print(self.clf.fit(x,y).feature_importances_)\n",
    "#\"-----------------定义oof:stacking的核心流程-------------\"\n",
    "print(\"-------->>>>>定义oof:stacking的核心流程<<<<<--------\")\n",
    "'''\n",
    "'''\n",
    "def get_oof(clf, x_train, y_train, x_test):\n",
    "    oof_train = np.zeros((ntrain,))#(14865,)\n",
    "    oof_test = np.zeros((ntest,))#(10000,)\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))#(5, 10000)\n",
    "    #\n",
    "    fold=0\n",
    "    for train_index, test_index in sk.split(x_train, y_train):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]#该折余下的验证集\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict_proba(x_te)#对该折余下的验证集做预测，将结果填补在这些数据在原始数据中的位置\n",
    "        oof_test_skf[fold, :] = clf.predict_proba(x_test)#用此时的模型（第i折的模型）对测试做预测，放在第i折对应的位置\n",
    "        #oof_train[test_index] = clf.predict(x_te)\n",
    "        #oof_test_skf[fold, :] = clf.predict(x_test)\n",
    "        fold+=1\n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)#将这N折模型对测试集的预测结果进行一个平均，作为改模型的预测结果\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n",
    "#\n",
    "'''\n",
    "第一层的基模型\n",
    "'''\n",
    "# Put in our parameters for said classifiers\n",
    "# Random Forest parameters\n",
    "# \n",
    "print(\"-------->>>>>第一层的基模型<<<<<--------\")\n",
    "rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_v2_params)\n",
    "et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)\n",
    "gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)\n",
    "lgb= SklearnHelper(clf=lightgbm.LGBMClassifier, seed=SEED, params=lgb_params)\n",
    "xgb = SklearnHelper(clf=xgboost.XGBClassifier, seed=SEED, params=xgb_params)\n",
    "cab=SklearnHelper(clf=catboost.CatBoostClassifier, seed=SEED, params=cab_params)\n",
    "\n",
    "#\n",
    "#\n",
    "y_train = kind\n",
    "\n",
    "x_train = train_data.values # Creates an array of the train data\n",
    "x_test = test_data.values # Creats an array of the test data\n",
    "#\n",
    "print(\"-------->>>>>训练、测试数据<<<<<--------\")\n",
    "print(\"y_train:{};train_data:{};test_data:{}\".format(y_train.shape,train_data.shape,test_data.shape))\n",
    "#\n",
    "# Create our OOF train and test predictions. These base results will be used as new features\n",
    "print(\"-------->>>>>第一阶段训练oof<<<<<--------\")\n",
    "et_oof_train, et_oof_test = get_oof(et, x_train, y_train, x_test) # Extra Trees\n",
    "print(\"Extra Trees.............\")\n",
    "rf_oof_train, rf_oof_test = get_oof(rf,x_train, y_train, x_test) # Random Forest\n",
    "print(\"Random Forest.............\")\n",
    "gb_oof_train, gb_oof_test = get_oof(gb,x_train, y_train, x_test) # Gradient Boost\n",
    "print(\"Gradient.............\")\n",
    "lgb_oof_train, lgb_oof_test = get_oof(lgb,x_train, y_train, x_test) # Gradient Boost\n",
    "print(\"LGBClassifier.............\")\n",
    "xgb_oof_train, xgb_oof_test = get_oof(xgb,x_train, y_train, x_test) # XGBClassifier\n",
    "print(\"XGBClassifier.............\")\n",
    "cab_oof_train, cab_oof_test = get_oof(cab,x_train, y_train, x_test)# CatClassifier\n",
    "print(\"CatClassifier.............\")\n",
    "\n",
    "print(\"Training Stage_1 is complete\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# stage-2的调参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 80\n",
      "[80, 0.04, 4, 4] 0.8389775244206701\n",
      "[80, 0.05, 4, 4] 0.8399405566673567\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,num_leaves):\n",
    "    params={'num_leaves':num_leaves\n",
    "                   ,'max_depth':max_depth\n",
    "                   ,'learning_rate':lr\n",
    "                   ,'n_estimators':n_estimators\n",
    "                   ,'n_jobs':8\n",
    "               }\n",
    "    mean_f1=k_fold_serachParmaters(lightgbm.LGBMClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [80]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.04,0.05,0.06]:\n",
    "        for max_depth in [4,5,6,7,8]:\n",
    "            for num_leaves in [4,5,6,7,8]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,num_leaves)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,num_leaves]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[80, 0.05, 4, 4] 0.8419565786981478"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 50\n",
      "[50, 0.03, 3, 5] 0.8285658885103605\n",
      "[50, 0.03, 3, 7] 0.830231529550203\n",
      "[50, 0.03, 3, 9] 0.8316167765653166\n",
      "[50, 0.03, 3, 11] 0.8325856853741764\n",
      "[50, 0.03, 3, 13] 0.835269062766804\n"
     ]
    }
   ],
   "source": [
    "def search_param(n_estimators,lr,max_depth,min_child_weight):\n",
    "    params={'max_depth':max_depth\n",
    "              ,'learning_rate':lr\n",
    "              ,'n_estimators':n_estimators\n",
    "              ,'min_child_weight':min_child_weight\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "           }\n",
    "    \n",
    "    mean_f1=k_fold_serachParmaters(xgboost.XGBClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [50]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for lr in [0.03,0.04]:\n",
    "        for max_depth in [3,4,5,6]:\n",
    "            for min_child_weight in [5,7,9,11,13,15]:\n",
    "                mean_f1=search_param(n_estimators,lr,max_depth,min_child_weight)\n",
    "                if mean_f1>best:\n",
    "                    param=[n_estimators,lr,max_depth,min_child_weight]\n",
    "                    best=mean_f1\n",
    "                    print(param,best)#[50, 0.04, 4, 13] 0.8449774193716448"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter_cnt: 60\n",
      "[60, 0.03, 4] 0.8368955331348376\n",
      "[60, 0.04, 5] 0.8373802817468526\n",
      "[60, 0.05, 5] 0.8375477186456622\n",
      "iter_cnt: 55\n",
      "[60, 0.05, 5] 0.8375477186456622\n"
     ]
    }
   ],
   "source": [
    "def search_param(iter_cnt,lr,max_depth):\n",
    "    \n",
    "    params={'iterations':iter_cnt\n",
    "              ,'learning_rate':lr\n",
    "              ,'depth':max_depth\n",
    "              ,'silent':True\n",
    "              ,'thread_count':8\n",
    "              ,'task_type':'CPU'\n",
    "        }\n",
    "    mean_f1=k_fold_serachParmaters(catboost.CatBoostClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数[52, 0.073, 7] 0.8440488273317126\n",
    "param=[]\n",
    "best=0\n",
    "for iter_cnt in [60,55]:\n",
    "    print('iter_cnt:',iter_cnt)\n",
    "    for lr in [0.03,0.04,0.05]:\n",
    "        for max_depth in [4,5,6]:\n",
    "            mean_f1=search_param(iter_cnt,lr,max_depth)\n",
    "            if mean_f1>best:\n",
    "                param=[iter_cnt,lr,max_depth]#[55, 0.04, 5] 0.8475966668074264\n",
    "                best=mean_f1\n",
    "                print(param,best)\n",
    "print(param,best)#[55, 0.04, 5] 0.8475966668074264\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators: 40\n",
      "[40, 4, 4] 0.8314564206726864\n",
      "[40, 5, 4] 0.8334650927578162\n",
      "[40, 5, 5] 0.8346889391653527\n",
      "[40, 5, 7] 0.8352733404040108\n",
      "[40, 5, 8] 0.8364672803657551\n",
      "n_estimators: 50\n",
      "[50, 5, 4] 0.8365538263042755\n",
      "[50, 5, 7] 0.8371252073213781\n"
     ]
    }
   ],
   "source": [
    "# #\n",
    "def search_param(n_estimators,max_depth,min_samples_split):\n",
    "    params={'oob_score':True,\n",
    "            'random_state':2020,\n",
    "            'n_estimators': n_estimators,\n",
    "            'max_depth':max_depth,\n",
    "            'min_samples_split':min_samples_split\n",
    "            }\n",
    "    mean_f1=k_fold_serachParmaters(RandomForestClassifier(**params),pd.DataFrame(x_train),pd.DataFrame(y_train))\n",
    "    return mean_f1\n",
    "\n",
    "#搜索最佳参数\n",
    "param=[]\n",
    "best=0\n",
    "for n_estimators in [40,50]:\n",
    "    print('n_estimators:',n_estimators)\n",
    "    for min_samples_split in [4,5,7,8]:\n",
    "        for max_depth in [4,5,7,9,]:\n",
    "            mean_f1=search_param(n_estimators,max_depth,min_samples_split)\n",
    "            if mean_f1>best:\n",
    "                param=[n_estimators,max_depth,min_samples_split]\n",
    "                best=mean_f1\n",
    "                print(param,best)#[50, 4, 8] 0.8442795786278773\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LGBMClassifier: 0.8241452460027052\n",
      "XGBClassifier: 0.820427862402487\n",
      "CatBoostClassifier: 0.8191138393920616\n",
      "RandomForestClassifier_v1: 0.8215444728633191\n"
     ]
    }
   ],
   "source": [
    "#stage_2的模型调参\n",
    "\n",
    "lgb_params_stage2 = {'num_leaves':4\n",
    "                ,'max_depth':4\n",
    "                ,'learning_rate':0.05\n",
    "                ,'n_estimators':80\n",
    "                ,'n_jobs':8\n",
    "             }\n",
    "xgb_params_stage2 ={'max_depth':3\n",
    "              ,'learning_rate':0.03\n",
    "              ,'n_estimators':50\n",
    "             ,'min_child_weight':13\n",
    "              ,'n_jobs':8\n",
    "              ,'importance_type':'total_cover'\n",
    "            }\n",
    "cab_params_stage2={'iterations':60\n",
    "          ,'learning_rate':0.05\n",
    "          ,'depth':5\n",
    "          ,'silent':True\n",
    "          ,'thread_count':8\n",
    "          ,'task_type':'CPU'\n",
    "          #,'cat_features':cat_features\n",
    "}\n",
    "rf_params_stage2={'oob_score':True,\n",
    "            'random_state':2020,\n",
    "            'n_estimators': 50,\n",
    "            'max_depth':7,\n",
    "            'min_samples_split':7\n",
    "            }\n",
    "# #\n",
    "x_train = np.concatenate(( et_oof_train, rf_oof_train, gb_oof_train,lgb_oof_train, xgb_oof_train,cab_oof_train), axis=1)\n",
    "x_test = np.concatenate(( et_oof_test, rf_oof_test,gb_oof_test,lgb_oof_test, xgb_oof_test, cab_oof_test), axis=1)\n",
    "#\n",
    "lgb_stage2=lightgbm.LGBMClassifier(**lgb_params_stage2)\n",
    "print('LGBMClassifier:',k_fold_serachParmaters(lgb_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "xgb_stage2=xgboost.XGBClassifier(**xgb_params_stage2)\n",
    "print('XGBClassifier:',k_fold_serachParmaters(xgb_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "cab_stage2=catboost.CatBoostClassifier(**cab_params_stage2)\n",
    "print('CatBoostClassifier:',k_fold_serachParmaters(cab_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))\n",
    "rf_stage2 = RandomForestClassifier(**rf_params_stage2)\n",
    "print(\"RandomForestClassifier_v1:\",k_fold_serachParmaters(rf_stage2,pd.DataFrame(x_train),pd.DataFrame(y_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training is complete\n",
      "------第二阶段训练开始--------\n",
      "预测结果中合法的数量9079;违法的数量921,合法/违法9.857763\n",
      "预测结果中合法的数量9027;违法的数量973,合法/违法9.277492\n",
      "预测结果中合法的数量9067;违法的数量933,合法/违法9.718114\n",
      "预测结果中合法的数量9062;违法的数量938,合法/违法9.660981\n"
     ]
    }
   ],
   "source": [
    "def cnt_result(xx):\n",
    "    cnt_re={0:0,1:0}\n",
    "    for a in xx:\n",
    "        if a<=0.5:\n",
    "            cnt_re[0]+=1\n",
    "        else:\n",
    "            cnt_re[1]+=1\n",
    "    return cnt_re\n",
    "print(\"Training is complete\")\n",
    "print(\"------第二阶段训练开始--------\")\n",
    "#\n",
    "predictions_stage2=[]\n",
    "for model_two_stage in [xgb_stage2,cab_stage2,rf_stage2]:\n",
    "    #\n",
    "    model_two_stage.fit(x_train, y_train)\n",
    "    predictions = model_two_stage.predict_proba(x_test)[:,1]\n",
    "    predictions_stage2.append(predictions)\n",
    "    cnt_re=cnt_result(predictions)\n",
    "    print(\"预测结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))\n",
    "#\n",
    "predictions=sum(predictions_stage2)/len(predictions_stage2)#\n",
    "#predictions=np.sqrt(sum(np.array(np.array(predictions_stage2))**2)/len(predictions_stage2))#平方平均\n",
    "#predictions=pow(np.prod(np.array(predictions_stage2), axis=0),1/len(predictions_stage2))#几何平均\n",
    "cnt_re=cnt_result(predictions)\n",
    "print(\"预测结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "投票结果中合法的数量9066;违法的数量934,合法/违法9.706638\n"
     ]
    }
   ],
   "source": [
    "votes=[(pre>0.5).astype(int) for pre in predictions_stage2]\n",
    "vote_most=[]\n",
    "for i in range(len(predictions_stage2[0])):\n",
    "    vote_list=np.array(votes)[:,i]\n",
    "    tmp = {0: 0, 1: 0}\n",
    "    for k in vote_list:\n",
    "        tmp[k] += 1\n",
    "    #\n",
    "    most = sorted(tmp.items(), key=lambda item: item[1])[-1][0]\n",
    "    vote_most.append(most)\n",
    "cnt_re=cnt_result(vote_most)\n",
    "print(\"投票结果中合法的数量%d;违法的数量%d,合法/违法%f\"%(cnt_re[0],cnt_re[1],cnt_re[0]/cnt_re[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9988"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score((predictions>0.5).astype(int),vote_most)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../submission/submit_stack_cab_lgb_xgb_rf934.csv'"
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "test_df['score']=vote_most#predictions#可选:fina_persudo是伪标签的预测结果\n",
    "submit_csv=test_df[['id','score']]\n",
    "save_path='../submission/submit_stack_cab_lgb_xgb_rf'+str(cnt_re[1])+'.csv'\n",
    "submit_csv.to_csv(save_path,index=False)\n",
    "save_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3</td>\n",
       "      <td>0.017476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>da8691b210adb3f67820f5e0c87b337d63112cee52211888</td>\n",
       "      <td>0.017356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17</td>\n",
       "      <td>0.017639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db</td>\n",
       "      <td>0.018168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f000950527a6feb617e8d6ca7025dcf9d765429969122069</td>\n",
       "      <td>0.018279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325</td>\n",
       "      <td>0.019584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda</td>\n",
       "      <td>0.481104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7</td>\n",
       "      <td>0.017947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31</td>\n",
       "      <td>0.026300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a</td>\n",
       "      <td>0.032434</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    id     score\n",
       "0     9c7fa510616a683058ce97d0bc768a621cd85ab1e87da2a3  0.017476\n",
       "1     da8691b210adb3f67820f5e0c87b337d63112cee52211888  0.017356\n",
       "2     9c7fa510616a68309e4badf2a7a3123c0462fb85bf28ef17  0.017639\n",
       "3     f000950527a6feb6ed308bc4c7ae11276eab86480f8e03db  0.018168\n",
       "4     f000950527a6feb617e8d6ca7025dcf9d765429969122069  0.018279\n",
       "...                                                ...       ...\n",
       "9995  f1c1045b13d18329a2bd99d2a7e2227688c0d69bf1d1e325  0.019584\n",
       "9996  f000950527a6feb6bde38216d7cbbf32e66d3a3a96d4dbda  0.481104\n",
       "9997  da8691b210adb3f65b43370d3a362f4aa1d3b16b5ba0c9d7  0.017947\n",
       "9998  516ab81418ed215dcbbf0614a7b929e691f8eed153d7bb31  0.026300\n",
       "9999  9c7fa510616a68303d3427d4bfd4b0cf3e4843f2bf3f637a  0.032434\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huawei_copy",
   "language": "python",
   "name": "huawei_copy"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
